drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <ethdev_driver.h>
   6 #include <ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13 #include <rte_vect.h>
  14
  15 #include "fm10k.h"
  16 #include "base/fm10k_api.h"
  17
  18 /* Default delay to acquire mailbox lock */
  19 #define FM10K_MBXLOCK_DELAY_US 20
  20 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  21
  22 #define MAIN_VSI_POOL_NUMBER 0
  23
  24 /* Max try times to acquire switch status */
  25 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  26 /* Wait interval to get switch status */
  27 #define WAIT_SWITCH_MSG_US    100000
  28 /* A period of quiescence for switch */
  29 #define FM10K_SWITCH_QUIESCE_US 100000
  30 /* Number of chars per uint32 type */
  31 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  32 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  33
  34 /* default 1:1 map from queue ID to interrupt vector ID */
  35 #define Q2V(pci_dev, queue_id)                                                 \
  36         (rte_intr_vec_list_index_get((pci_dev)->intr_handle, queue_id))
  37
  38 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  39 #define MAX_LPORT_NUM    128
  40 #define GLORT_FD_Q_BASE  0x40
  41 #define GLORT_PF_MASK    0xFFC0
  42 #define GLORT_FD_MASK    GLORT_PF_MASK
  43 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  44
  45 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  46 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  47 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  48 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  49 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  50 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  51 static int
  52 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  53 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  54         const u8 *mac, bool add, uint32_t pool);
  55 static void fm10k_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
  56 static void fm10k_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
  57 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  58 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  59 static int fm10k_check_ftag(struct rte_devargs *devargs);
  60 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  61
  62 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  63                                struct rte_eth_dev_info *dev_info);
  64 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  65 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  68
  69 struct fm10k_xstats_name_off {
  70         char name[RTE_ETH_XSTATS_NAME_SIZE];
  71         unsigned offset;
  72 };
  73
  74 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  75         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  76         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  77         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  78         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  79         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  80         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  81         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  82         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  83                 nodesc_drop)},
  84 };
  85
  86 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  87                 sizeof(fm10k_hw_stats_strings[0]))
  88
  89 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  90         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  91         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  92         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  93 };
  94
  95 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  96                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  97
  98 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
  99         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 100         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 101 };
 102
 103 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 104                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 105
 106 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 107                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 108 static int
 109 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 110
 111 static void
 112 fm10k_mbx_initlock(struct fm10k_hw *hw)
 113 {
 114         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 115 }
 116
 117 static void
 118 fm10k_mbx_lock(struct fm10k_hw *hw)
 119 {
 120         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 121                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 122 }
 123
 124 static void
 125 fm10k_mbx_unlock(struct fm10k_hw *hw)
 126 {
 127         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 128 }
 129
 130 /* Stubs needed for linkage when vPMD is disabled */
 131 __rte_weak int
 132 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 133 {
 134         return -1;
 135 }
 136
 137 __rte_weak uint16_t
 138 fm10k_recv_pkts_vec(
 139         __rte_unused void *rx_queue,
 140         __rte_unused struct rte_mbuf **rx_pkts,
 141         __rte_unused uint16_t nb_pkts)
 142 {
 143         return 0;
 144 }
 145
 146 __rte_weak uint16_t
 147 fm10k_recv_scattered_pkts_vec(
 148                 __rte_unused void *rx_queue,
 149                 __rte_unused struct rte_mbuf **rx_pkts,
 150                 __rte_unused uint16_t nb_pkts)
 151 {
 152         return 0;
 153 }
 154
 155 __rte_weak int
 156 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 157
 158 {
 159         return -1;
 160 }
 161
 162 __rte_weak void
 163 fm10k_rx_queue_release_mbufs_vec(
 164                 __rte_unused struct fm10k_rx_queue *rxq)
 165 {
 166         return;
 167 }
 168
 169 __rte_weak void
 170 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 171 {
 172         return;
 173 }
 174
 175 __rte_weak int
 176 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 177 {
 178         return -1;
 179 }
 180
 181 __rte_weak uint16_t
 182 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 183                            __rte_unused struct rte_mbuf **tx_pkts,
 184                            __rte_unused uint16_t nb_pkts)
 185 {
 186         return 0;
 187 }
 188
 189 /*
 190  * reset queue to initial state, allocate software buffers used when starting
 191  * device.
 192  * return 0 on success
 193  * return -ENOMEM if buffers cannot be allocated
 194  * return -EINVAL if buffers do not satisfy alignment condition
 195  */
 196 static inline int
 197 rx_queue_reset(struct fm10k_rx_queue *q)
 198 {
 199         static const union fm10k_rx_desc zero = {{0} };
 200         uint64_t dma_addr;
 201         int i, diag;
 202         PMD_INIT_FUNC_TRACE();
 203
 204         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 205         if (diag != 0)
 206                 return -ENOMEM;
 207
 208         for (i = 0; i < q->nb_desc; ++i) {
 209                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 210                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 211                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 212                                                 q->nb_desc);
 213                         return -EINVAL;
 214                 }
 215                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 216                 q->hw_ring[i].q.pkt_addr = dma_addr;
 217                 q->hw_ring[i].q.hdr_addr = dma_addr;
 218         }
 219
 220         /* initialize extra software ring entries. Space for these extra
 221          * entries is always allocated.
 222          */
 223         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 224         for (i = 0; i < q->nb_fake_desc; ++i) {
 225                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 226                 q->hw_ring[q->nb_desc + i] = zero;
 227         }
 228
 229         q->next_dd = 0;
 230         q->next_alloc = 0;
 231         q->next_trigger = q->alloc_thresh - 1;
 232         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 233         q->rxrearm_start = 0;
 234         q->rxrearm_nb = 0;
 235
 236         return 0;
 237 }
 238
 239 /*
 240  * clean queue, descriptor rings, free software buffers used when stopping
 241  * device.
 242  */
 243 static inline void
 244 rx_queue_clean(struct fm10k_rx_queue *q)
 245 {
 246         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 247         uint32_t i;
 248         PMD_INIT_FUNC_TRACE();
 249
 250         /* zero descriptor rings */
 251         for (i = 0; i < q->nb_desc; ++i)
 252                 q->hw_ring[i] = zero;
 253
 254         /* zero faked descriptors */
 255         for (i = 0; i < q->nb_fake_desc; ++i)
 256                 q->hw_ring[q->nb_desc + i] = zero;
 257
 258         /* vPMD has a different way of releasing mbufs. */
 259         if (q->rx_using_sse) {
 260                 fm10k_rx_queue_release_mbufs_vec(q);
 261                 return;
 262         }
 263
 264         /* free software buffers */
 265         for (i = 0; i < q->nb_desc; ++i) {
 266                 if (q->sw_ring[i]) {
 267                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 268                         q->sw_ring[i] = NULL;
 269                 }
 270         }
 271 }
 272
 273 /*
 274  * free all queue memory used when releasing the queue (i.e. configure)
 275  */
 276 static inline void
 277 rx_queue_free(struct fm10k_rx_queue *q)
 278 {
 279         PMD_INIT_FUNC_TRACE();
 280         if (q) {
 281                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 282                 rx_queue_clean(q);
 283                 if (q->sw_ring) {
 284                         rte_free(q->sw_ring);
 285                         q->sw_ring = NULL;
 286                 }
 287                 rte_free(q);
 288                 q = NULL;
 289         }
 290 }
 291
 292 /*
 293  * disable RX queue, wait unitl HW finished necessary flush operation
 294  */
 295 static inline int
 296 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 297 {
 298         uint32_t reg, i;
 299
 300         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 301         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 302                         reg & ~FM10K_RXQCTL_ENABLE);
 303
 304         /* Wait 100us at most */
 305         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 306                 rte_delay_us(1);
 307                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 308                 if (!(reg & FM10K_RXQCTL_ENABLE))
 309                         break;
 310         }
 311
 312         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 313                 return -1;
 314
 315         return 0;
 316 }
 317
 318 /*
 319  * reset queue to initial state, allocate software buffers used when starting
 320  * device
 321  */
 322 static inline void
 323 tx_queue_reset(struct fm10k_tx_queue *q)
 324 {
 325         PMD_INIT_FUNC_TRACE();
 326         q->last_free = 0;
 327         q->next_free = 0;
 328         q->nb_used = 0;
 329         q->nb_free = q->nb_desc - 1;
 330         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 331         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 332 }
 333
 334 /*
 335  * clean queue, descriptor rings, free software buffers used when stopping
 336  * device
 337  */
 338 static inline void
 339 tx_queue_clean(struct fm10k_tx_queue *q)
 340 {
 341         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 342         uint32_t i;
 343         PMD_INIT_FUNC_TRACE();
 344
 345         /* zero descriptor rings */
 346         for (i = 0; i < q->nb_desc; ++i)
 347                 q->hw_ring[i] = zero;
 348
 349         /* free software buffers */
 350         for (i = 0; i < q->nb_desc; ++i) {
 351                 if (q->sw_ring[i]) {
 352                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 353                         q->sw_ring[i] = NULL;
 354                 }
 355         }
 356 }
 357
 358 /*
 359  * free all queue memory used when releasing the queue (i.e. configure)
 360  */
 361 static inline void
 362 tx_queue_free(struct fm10k_tx_queue *q)
 363 {
 364         PMD_INIT_FUNC_TRACE();
 365         if (q) {
 366                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 367                 tx_queue_clean(q);
 368                 if (q->rs_tracker.list) {
 369                         rte_free(q->rs_tracker.list);
 370                         q->rs_tracker.list = NULL;
 371                 }
 372                 if (q->sw_ring) {
 373                         rte_free(q->sw_ring);
 374                         q->sw_ring = NULL;
 375                 }
 376                 rte_free(q);
 377                 q = NULL;
 378         }
 379 }
 380
 381 /*
 382  * disable TX queue, wait unitl HW finished necessary flush operation
 383  */
 384 static inline int
 385 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 386 {
 387         uint32_t reg, i;
 388
 389         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 390         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 391                         reg & ~FM10K_TXDCTL_ENABLE);
 392
 393         /* Wait 100us at most */
 394         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 395                 rte_delay_us(1);
 396                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 397                 if (!(reg & FM10K_TXDCTL_ENABLE))
 398                         break;
 399         }
 400
 401         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 402                 return -1;
 403
 404         return 0;
 405 }
 406
 407 static int
 408 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 409 {
 410         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 411         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 412         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 413         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 414
 415         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 416
 417         if (rx_mq_mode & RTE_ETH_MQ_RX_DCB_FLAG) {
 418                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 419                 return -EINVAL;
 420         }
 421
 422         if (!(rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_FLAG))
 423                 return 0;
 424
 425         if (hw->mac.type == fm10k_mac_vf) {
 426                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 427                 return -EINVAL;
 428         }
 429
 430         /* Check VMDQ queue pool number */
 431         if (vmdq_conf->nb_queue_pools >
 432                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 433                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 434                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 435                         vmdq_conf->nb_queue_pools);
 436                 return -EINVAL;
 437         }
 438
 439         return 0;
 440 }
 441
 442 static const struct fm10k_txq_ops def_txq_ops = {
 443         .reset = tx_queue_reset,
 444 };
 445
 446 static int
 447 fm10k_dev_configure(struct rte_eth_dev *dev)
 448 {
 449         int ret;
 450
 451         PMD_INIT_FUNC_TRACE();
 452
 453         if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
 454                 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
 455
 456         /* multipe queue mode checking */
 457         ret  = fm10k_check_mq_mode(dev);
 458         if (ret != 0) {
 459                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 460                             ret);
 461                 return ret;
 462         }
 463
 464         dev->data->scattered_rx = 0;
 465
 466         return 0;
 467 }
 468
 469 static void
 470 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 471 {
 472         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 473         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 474         uint32_t i;
 475
 476         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 477
 478         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 479                 if (!vmdq_conf->pool_map[i].pools)
 480                         continue;
 481                 fm10k_mbx_lock(hw);
 482                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 483                 fm10k_mbx_unlock(hw);
 484         }
 485 }
 486
 487 static void
 488 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 489 {
 490         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 491
 492         /* Add default mac address */
 493         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 494                 MAIN_VSI_POOL_NUMBER);
 495 }
 496
 497 static void
 498 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 499 {
 500         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 501         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 502         uint32_t mrqc, *key, i, reta, j;
 503         uint64_t hf;
 504
 505 #define RSS_KEY_SIZE 40
 506         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 507                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 508                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 509                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 510                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 511                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 512         };
 513
 514         if (dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_RSS ||
 515                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 516                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 517                 return;
 518         }
 519
 520         /* random key is rss_intel_key (default) or user provided (rss_key) */
 521         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 522                 key = (uint32_t *)rss_intel_key;
 523         else
 524                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 525
 526         /* Now fill our hash function seeds, 4 bytes at a time */
 527         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 528                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 529
 530         /*
 531          * Fill in redirection table
 532          * The byte-swap is needed because NIC registers are in
 533          * little-endian order.
 534          */
 535         reta = 0;
 536         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 537                 if (j == dev->data->nb_rx_queues)
 538                         j = 0;
 539                 reta = (reta << CHAR_BIT) | j;
 540                 if ((i & 3) == 3)
 541                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 542                                         rte_bswap32(reta));
 543         }
 544
 545         /*
 546          * Generate RSS hash based on packet types, TCP/UDP
 547          * port numbers and/or IPv4/v6 src and dst addresses
 548          */
 549         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 550         mrqc = 0;
 551         mrqc |= (hf & RTE_ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 552         mrqc |= (hf & RTE_ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 553         mrqc |= (hf & RTE_ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 554         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 555         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 556         mrqc |= (hf & RTE_ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 557         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 558         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 559         mrqc |= (hf & RTE_ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 560
 561         if (mrqc == 0) {
 562                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 563                         "supported", hf);
 564                 return;
 565         }
 566
 567         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 568 }
 569
 570 static void
 571 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 572 {
 573         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 574         uint32_t i;
 575
 576         for (i = 0; i < nb_lport_new; i++) {
 577                 /* Set unicast mode by default. App can change
 578                  * to other mode in other API func.
 579                  */
 580                 fm10k_mbx_lock(hw);
 581                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 582                         FM10K_XCAST_MODE_NONE);
 583                 fm10k_mbx_unlock(hw);
 584         }
 585 }
 586
 587 static void
 588 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 589 {
 590         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 591         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 592         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 593         struct fm10k_macvlan_filter_info *macvlan;
 594         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 595         uint16_t nb_lport_new;
 596
 597         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 598         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 599
 600         fm10k_dev_rss_configure(dev);
 601
 602         /* only PF supports VMDQ */
 603         if (hw->mac.type != fm10k_mac_pf)
 604                 return;
 605
 606         if (dev_conf->rxmode.mq_mode & RTE_ETH_MQ_RX_VMDQ_FLAG)
 607                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 608
 609         /* no pool number change, no need to update logic port and VLAN/MAC */
 610         if (macvlan->nb_queue_pools == nb_queue_pools)
 611                 return;
 612
 613         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 614         fm10k_dev_logic_port_update(dev, nb_lport_new);
 615
 616         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 617         memset(dev->data->mac_addrs, 0,
 618                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 619         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 620                 &dev->data->mac_addrs[0]);
 621         memset(macvlan, 0, sizeof(*macvlan));
 622         macvlan->nb_queue_pools = nb_queue_pools;
 623
 624         if (nb_queue_pools)
 625                 fm10k_dev_vmdq_rx_configure(dev);
 626         else
 627                 fm10k_dev_pf_main_vsi_reset(dev);
 628 }
 629
 630 static int
 631 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 632 {
 633         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 634         int i, ret;
 635         struct fm10k_tx_queue *txq;
 636         uint64_t base_addr;
 637         uint32_t size;
 638
 639         /* Disable TXINT to avoid possible interrupt */
 640         for (i = 0; i < hw->mac.max_queues; i++)
 641                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 642                                 3 << FM10K_TXINT_TIMER_SHIFT);
 643
 644         /* Setup TX queue */
 645         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 646                 txq = dev->data->tx_queues[i];
 647                 base_addr = txq->hw_ring_phys_addr;
 648                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 649
 650                 /* disable queue to avoid issues while updating state */
 651                 ret = tx_queue_disable(hw, i);
 652                 if (ret) {
 653                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 654                         return -1;
 655                 }
 656                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 657                  * register is read-only for VF.
 658                  */
 659                 if (fm10k_check_ftag(dev->device->devargs)) {
 660                         if (hw->mac.type == fm10k_mac_pf) {
 661                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 662                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 663                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 664                         } else {
 665                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 666                                 return -ENOTSUP;
 667                         }
 668                 }
 669
 670                 /* set location and size for descriptor ring */
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 672                                 base_addr & UINT64_LOWER_32BITS_MASK);
 673                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 674                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 675                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 676
 677                 /* assign default SGLORT for each TX queue by PF */
 678                 if (hw->mac.type == fm10k_mac_pf)
 679                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 680         }
 681
 682         /* set up vector or scalar TX function as appropriate */
 683         fm10k_set_tx_function(dev);
 684
 685         return 0;
 686 }
 687
 688 static int
 689 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 690 {
 691         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 692         struct fm10k_macvlan_filter_info *macvlan;
 693         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 694         struct rte_intr_handle *intr_handle = pdev->intr_handle;
 695         int i, ret;
 696         struct fm10k_rx_queue *rxq;
 697         uint64_t base_addr;
 698         uint32_t size;
 699         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 700         uint32_t logic_port = hw->mac.dglort_map;
 701         uint16_t buf_size;
 702         uint16_t queue_stride = 0;
 703
 704         /* enable RXINT for interrupt mode */
 705         i = 0;
 706         if (rte_intr_dp_is_en(intr_handle)) {
 707                 for (; i < dev->data->nb_rx_queues; i++) {
 708                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 709                         if (hw->mac.type == fm10k_mac_pf)
 710                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 711                                         FM10K_ITR_AUTOMASK |
 712                                         FM10K_ITR_MASK_CLEAR);
 713                         else
 714                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 715                                         FM10K_ITR_AUTOMASK |
 716                                         FM10K_ITR_MASK_CLEAR);
 717                 }
 718         }
 719         /* Disable other RXINT to avoid possible interrupt */
 720         for (; i < hw->mac.max_queues; i++)
 721                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 722                         3 << FM10K_RXINT_TIMER_SHIFT);
 723
 724         /* Setup RX queues */
 725         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 726                 rxq = dev->data->rx_queues[i];
 727                 base_addr = rxq->hw_ring_phys_addr;
 728                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 729
 730                 /* disable queue to avoid issues while updating state */
 731                 ret = rx_queue_disable(hw, i);
 732                 if (ret) {
 733                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 734                         return -1;
 735                 }
 736
 737                 /* Setup the Base and Length of the Rx Descriptor Ring */
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 739                                 base_addr & UINT64_LOWER_32BITS_MASK);
 740                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 741                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 742                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 743
 744                 /* Configure the Rx buffer size for one buff without split */
 745                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 746                         RTE_PKTMBUF_HEADROOM);
 747                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 748                  * reserved for this purpose, and the worst case could be 511B.
 749                  * But SRR reg assumes all buffers have the same size. In order
 750                  * to fill the gap, we'll have to consider the worst case and
 751                  * assume 512B is reserved. If we don't do so, it's possible
 752                  * for HW to overwrite data to next mbuf.
 753                  */
 754                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 755
 756                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 757                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 758                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 759
 760                 /* It adds dual VLAN length for supporting dual VLAN */
 761                 if ((dev->data->mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN +
 762                                 2 * RTE_VLAN_HLEN) > buf_size ||
 763                         rxq->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) {
 764                         uint32_t reg;
 765                         dev->data->scattered_rx = 1;
 766                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 767                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 768                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 769                 }
 770
 771                 /* Enable drop on empty, it's RO for VF */
 772                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 773                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 774
 775                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 776                 FM10K_WRITE_FLUSH(hw);
 777         }
 778
 779         /* Configure VMDQ/RSS if applicable */
 780         fm10k_dev_mq_rx_configure(dev);
 781
 782         /* Decide the best RX function */
 783         fm10k_set_rx_function(dev);
 784
 785         /* update RX_SGLORT for loopback suppress*/
 786         if (hw->mac.type != fm10k_mac_pf)
 787                 return 0;
 788         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 789         if (macvlan->nb_queue_pools)
 790                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 791         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 792                 if (i && queue_stride && !(i % queue_stride))
 793                         logic_port++;
 794                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 795         }
 796
 797         return 0;
 798 }
 799
 800 static int
 801 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 802 {
 803         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 804         int err;
 805         uint32_t reg;
 806         struct fm10k_rx_queue *rxq;
 807
 808         PMD_INIT_FUNC_TRACE();
 809
 810         rxq = dev->data->rx_queues[rx_queue_id];
 811         err = rx_queue_reset(rxq);
 812         if (err == -ENOMEM) {
 813                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 814                 return err;
 815         } else if (err == -EINVAL) {
 816                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 817                         " %d", err);
 818                 return err;
 819         }
 820
 821         /* Setup the HW Rx Head and Tail Descriptor Pointers
 822          * Note: this must be done AFTER the queue is enabled on real
 823          * hardware, but BEFORE the queue is enabled when using the
 824          * emulation platform. Do it in both places for now and remove
 825          * this comment and the following two register writes when the
 826          * emulation platform is no longer being used.
 827          */
 828         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 829         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 830
 831         /* Set PF ownership flag for PF devices */
 832         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 833         if (hw->mac.type == fm10k_mac_pf)
 834                 reg |= FM10K_RXQCTL_PF;
 835         reg |= FM10K_RXQCTL_ENABLE;
 836         /* enable RX queue */
 837         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 838         FM10K_WRITE_FLUSH(hw);
 839
 840         /* Setup the HW Rx Head and Tail Descriptor Pointers
 841          * Note: this must be done AFTER the queue is enabled
 842          */
 843         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 844         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 845         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 846
 847         return 0;
 848 }
 849
 850 static int
 851 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 852 {
 853         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 854
 855         PMD_INIT_FUNC_TRACE();
 856
 857         /* Disable RX queue */
 858         rx_queue_disable(hw, rx_queue_id);
 859
 860         /* Free mbuf and clean HW ring */
 861         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 862         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 863
 864         return 0;
 865 }
 866
 867 static int
 868 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 869 {
 870         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 871         /** @todo - this should be defined in the shared code */
 872 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 873         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 874         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 875
 876         PMD_INIT_FUNC_TRACE();
 877
 878         q->ops->reset(q);
 879
 880         /* reset head and tail pointers */
 881         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 882         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 883
 884         /* enable TX queue */
 885         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 886                                 FM10K_TXDCTL_ENABLE | txdctl);
 887         FM10K_WRITE_FLUSH(hw);
 888         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 889
 890         return 0;
 891 }
 892
 893 static int
 894 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 895 {
 896         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 897
 898         PMD_INIT_FUNC_TRACE();
 899
 900         tx_queue_disable(hw, tx_queue_id);
 901         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 902         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 903
 904         return 0;
 905 }
 906
 907 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 908 {
 909         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 910                 != FM10K_DGLORTMAP_NONE);
 911 }
 912
 913 static int
 914 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 915 {
 916         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 917         int status;
 918
 919         PMD_INIT_FUNC_TRACE();
 920
 921         /* Return if it didn't acquire valid glort range */
 922         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 923                 return 0;
 924
 925         fm10k_mbx_lock(hw);
 926         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 927                                 FM10K_XCAST_MODE_PROMISC);
 928         fm10k_mbx_unlock(hw);
 929
 930         if (status != FM10K_SUCCESS) {
 931                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 932                 return -EAGAIN;
 933         }
 934
 935         return 0;
 936 }
 937
 938 static int
 939 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 940 {
 941         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 942         uint8_t mode;
 943         int status;
 944
 945         PMD_INIT_FUNC_TRACE();
 946
 947         /* Return if it didn't acquire valid glort range */
 948         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 949                 return 0;
 950
 951         if (dev->data->all_multicast == 1)
 952                 mode = FM10K_XCAST_MODE_ALLMULTI;
 953         else
 954                 mode = FM10K_XCAST_MODE_NONE;
 955
 956         fm10k_mbx_lock(hw);
 957         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 958                                 mode);
 959         fm10k_mbx_unlock(hw);
 960
 961         if (status != FM10K_SUCCESS) {
 962                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 963                 return -EAGAIN;
 964         }
 965
 966         return 0;
 967 }
 968
 969 static int
 970 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 971 {
 972         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return 0;
 980
 981         /* If promiscuous mode is enabled, it doesn't make sense to enable
 982          * allmulticast and disable promiscuous since fm10k only can select
 983          * one of the modes.
 984          */
 985         if (dev->data->promiscuous) {
 986                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 987                         "needn't enable allmulticast");
 988                 return 0;
 989         }
 990
 991         fm10k_mbx_lock(hw);
 992         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 993                                 FM10K_XCAST_MODE_ALLMULTI);
 994         fm10k_mbx_unlock(hw);
 995
 996         if (status != FM10K_SUCCESS) {
 997                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 998                 return -EAGAIN;
 999         }
1000
1001         return 0;
1002 }
1003
1004 static int
1005 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1006 {
1007         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1008         int status;
1009
1010         PMD_INIT_FUNC_TRACE();
1011
1012         /* Return if it didn't acquire valid glort range */
1013         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1014                 return 0;
1015
1016         if (dev->data->promiscuous) {
1017                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1018                         "since promisc mode is enabled");
1019                 return -EINVAL;
1020         }
1021
1022         fm10k_mbx_lock(hw);
1023         /* Change mode to unicast mode */
1024         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1025                                 FM10K_XCAST_MODE_NONE);
1026         fm10k_mbx_unlock(hw);
1027
1028         if (status != FM10K_SUCCESS) {
1029                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1030                 return -EAGAIN;
1031         }
1032
1033         return 0;
1034 }
1035
1036 static void
1037 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1038 {
1039         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1040         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1041         uint16_t nb_queue_pools;
1042         struct fm10k_macvlan_filter_info *macvlan;
1043
1044         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1045         nb_queue_pools = macvlan->nb_queue_pools;
1046         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1047         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1048
1049         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1050         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1051         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1052                         hw->mac.dglort_map;
1053         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1054         /* Configure VMDQ/RSS DGlort Decoder */
1055         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1056
1057         /* Flow Director configurations, only queue number is valid. */
1058         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1059         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1060                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1061         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1062         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1063
1064         /* Invalidate all other GLORT entries */
1065         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1066                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1067                                 FM10K_DGLORTMAP_NONE);
1068 }
1069
1070 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1071 static int
1072 fm10k_dev_start(struct rte_eth_dev *dev)
1073 {
1074         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1075         int i, diag;
1076
1077         PMD_INIT_FUNC_TRACE();
1078
1079         /* stop, init, then start the hw */
1080         diag = fm10k_stop_hw(hw);
1081         if (diag != FM10K_SUCCESS) {
1082                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1083                 return -EIO;
1084         }
1085
1086         diag = fm10k_init_hw(hw);
1087         if (diag != FM10K_SUCCESS) {
1088                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1089                 return -EIO;
1090         }
1091
1092         diag = fm10k_start_hw(hw);
1093         if (diag != FM10K_SUCCESS) {
1094                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1095                 return -EIO;
1096         }
1097
1098         diag = fm10k_dev_tx_init(dev);
1099         if (diag) {
1100                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1101                 return diag;
1102         }
1103
1104         if (fm10k_dev_rxq_interrupt_setup(dev))
1105                 return -EIO;
1106
1107         diag = fm10k_dev_rx_init(dev);
1108         if (diag) {
1109                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1110                 return diag;
1111         }
1112
1113         if (hw->mac.type == fm10k_mac_pf)
1114                 fm10k_dev_dglort_map_configure(dev);
1115
1116         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1117                 struct fm10k_rx_queue *rxq;
1118                 rxq = dev->data->rx_queues[i];
1119
1120                 if (rxq->rx_deferred_start)
1121                         continue;
1122                 diag = fm10k_dev_rx_queue_start(dev, i);
1123                 if (diag != 0) {
1124                         int j;
1125                         for (j = 0; j < i; ++j)
1126                                 rx_queue_clean(dev->data->rx_queues[j]);
1127                         return diag;
1128                 }
1129         }
1130
1131         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1132                 struct fm10k_tx_queue *txq;
1133                 txq = dev->data->tx_queues[i];
1134
1135                 if (txq->tx_deferred_start)
1136                         continue;
1137                 diag = fm10k_dev_tx_queue_start(dev, i);
1138                 if (diag != 0) {
1139                         int j;
1140                         for (j = 0; j < i; ++j)
1141                                 tx_queue_clean(dev->data->tx_queues[j]);
1142                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1143                                 rx_queue_clean(dev->data->rx_queues[j]);
1144                         return diag;
1145                 }
1146         }
1147
1148         /* Update default vlan when not in VMDQ mode */
1149         if (!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_VMDQ_FLAG))
1150                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1151
1152         fm10k_link_update(dev, 0);
1153
1154         return 0;
1155 }
1156
1157 static int
1158 fm10k_dev_stop(struct rte_eth_dev *dev)
1159 {
1160         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1161         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1162         struct rte_intr_handle *intr_handle = pdev->intr_handle;
1163         int i;
1164
1165         PMD_INIT_FUNC_TRACE();
1166         dev->data->dev_started = 0;
1167
1168         if (dev->data->tx_queues)
1169                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1170                         fm10k_dev_tx_queue_stop(dev, i);
1171
1172         if (dev->data->rx_queues)
1173                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1174                         fm10k_dev_rx_queue_stop(dev, i);
1175
1176         /* Disable datapath event */
1177         if (rte_intr_dp_is_en(intr_handle)) {
1178                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1179                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1180                                 3 << FM10K_RXINT_TIMER_SHIFT);
1181                         if (hw->mac.type == fm10k_mac_pf)
1182                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1183                                         FM10K_ITR_MASK_SET);
1184                         else
1185                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1186                                         FM10K_ITR_MASK_SET);
1187                 }
1188         }
1189         /* Clean datapath event and queue/vec mapping */
1190         rte_intr_efd_disable(intr_handle);
1191         rte_intr_vec_list_free(intr_handle);
1192
1193         return 0;
1194 }
1195
1196 static void
1197 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1198 {
1199         int i;
1200
1201         PMD_INIT_FUNC_TRACE();
1202
1203         if (dev->data->tx_queues) {
1204                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1205                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1206
1207                         tx_queue_free(txq);
1208                 }
1209         }
1210
1211         if (dev->data->rx_queues) {
1212                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1213                         fm10k_rx_queue_release(dev, i);
1214         }
1215 }
1216
1217 static int
1218 fm10k_link_update(struct rte_eth_dev *dev,
1219         __rte_unused int wait_to_complete)
1220 {
1221         struct fm10k_dev_info *dev_info =
1222                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1223         PMD_INIT_FUNC_TRACE();
1224
1225         dev->data->dev_link.link_speed  = RTE_ETH_SPEED_NUM_50G;
1226         dev->data->dev_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1227         dev->data->dev_link.link_status =
1228                 dev_info->sm_down ? RTE_ETH_LINK_DOWN : RTE_ETH_LINK_UP;
1229         dev->data->dev_link.link_autoneg = RTE_ETH_LINK_FIXED;
1230
1231         return 0;
1232 }
1233
1234 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1235         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1236 {
1237         unsigned i, q;
1238         unsigned count = 0;
1239
1240         if (xstats_names != NULL) {
1241                 /* Note: limit checked in rte_eth_xstats_names() */
1242
1243                 /* Global stats */
1244                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1245                         snprintf(xstats_names[count].name,
1246                                 sizeof(xstats_names[count].name),
1247                                 "%s", fm10k_hw_stats_strings[count].name);
1248                         count++;
1249                 }
1250
1251                 /* PF queue stats */
1252                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1253                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1254                                 snprintf(xstats_names[count].name,
1255                                         sizeof(xstats_names[count].name),
1256                                         "rx_q%u_%s", q,
1257                                         fm10k_hw_stats_rx_q_strings[i].name);
1258                                 count++;
1259                         }
1260                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1261                                 snprintf(xstats_names[count].name,
1262                                         sizeof(xstats_names[count].name),
1263                                         "tx_q%u_%s", q,
1264                                         fm10k_hw_stats_tx_q_strings[i].name);
1265                                 count++;
1266                         }
1267                 }
1268         }
1269         return FM10K_NB_XSTATS;
1270 }
1271
1272 static int
1273 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1274                  unsigned n)
1275 {
1276         struct fm10k_hw_stats *hw_stats =
1277                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1278         unsigned i, q, count = 0;
1279
1280         if (n < FM10K_NB_XSTATS)
1281                 return FM10K_NB_XSTATS;
1282
1283         /* Global stats */
1284         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1285                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1286                         fm10k_hw_stats_strings[count].offset);
1287                 xstats[count].id = count;
1288                 count++;
1289         }
1290
1291         /* PF queue stats */
1292         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1293                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1294                         xstats[count].value =
1295                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1296                                 fm10k_hw_stats_rx_q_strings[i].offset);
1297                         xstats[count].id = count;
1298                         count++;
1299                 }
1300                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1301                         xstats[count].value =
1302                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1303                                 fm10k_hw_stats_tx_q_strings[i].offset);
1304                         xstats[count].id = count;
1305                         count++;
1306                 }
1307         }
1308
1309         return FM10K_NB_XSTATS;
1310 }
1311
1312 static int
1313 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1314 {
1315         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1316         struct fm10k_hw *hw =
1317                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1318         struct fm10k_hw_stats *hw_stats =
1319                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1320         int i;
1321
1322         PMD_INIT_FUNC_TRACE();
1323
1324         fm10k_update_hw_stats(hw, hw_stats);
1325
1326         ipackets = opackets = ibytes = obytes = imissed = 0;
1327         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1328                 (i < hw->mac.max_queues); ++i) {
1329                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1330                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1331                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1332                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1333                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1334                 ipackets += stats->q_ipackets[i];
1335                 opackets += stats->q_opackets[i];
1336                 ibytes   += stats->q_ibytes[i];
1337                 obytes   += stats->q_obytes[i];
1338                 imissed  += stats->q_errors[i];
1339         }
1340         stats->ipackets = ipackets;
1341         stats->opackets = opackets;
1342         stats->ibytes = ibytes;
1343         stats->obytes = obytes;
1344         stats->imissed = imissed;
1345         return 0;
1346 }
1347
1348 static int
1349 fm10k_stats_reset(struct rte_eth_dev *dev)
1350 {
1351         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1352         struct fm10k_hw_stats *hw_stats =
1353                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1354
1355         PMD_INIT_FUNC_TRACE();
1356
1357         memset(hw_stats, 0, sizeof(*hw_stats));
1358         fm10k_rebind_hw_stats(hw, hw_stats);
1359
1360         return 0;
1361 }
1362
1363 static int
1364 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1365         struct rte_eth_dev_info *dev_info)
1366 {
1367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1369
1370         PMD_INIT_FUNC_TRACE();
1371
1372         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1373         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1374         dev_info->max_rx_queues      = hw->mac.max_queues;
1375         dev_info->max_tx_queues      = hw->mac.max_queues;
1376         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1377         dev_info->max_hash_mac_addrs = 0;
1378         dev_info->max_vfs            = pdev->max_vfs;
1379         dev_info->vmdq_pool_base     = 0;
1380         dev_info->vmdq_queue_base    = 0;
1381         dev_info->max_vmdq_pools     = RTE_ETH_32_POOLS;
1382         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1383         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1384         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1385                                     dev_info->rx_queue_offload_capa;
1386         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1387         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1388                                     dev_info->tx_queue_offload_capa;
1389
1390         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1391         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1392         dev_info->flow_type_rss_offloads = RTE_ETH_RSS_IPV4 |
1393                                         RTE_ETH_RSS_IPV6 |
1394                                         RTE_ETH_RSS_IPV6_EX |
1395                                         RTE_ETH_RSS_NONFRAG_IPV4_TCP |
1396                                         RTE_ETH_RSS_NONFRAG_IPV6_TCP |
1397                                         RTE_ETH_RSS_IPV6_TCP_EX |
1398                                         RTE_ETH_RSS_NONFRAG_IPV4_UDP |
1399                                         RTE_ETH_RSS_NONFRAG_IPV6_UDP |
1400                                         RTE_ETH_RSS_IPV6_UDP_EX;
1401
1402         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1403                 .rx_thresh = {
1404                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1405                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1406                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1407                 },
1408                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1409                 .rx_drop_en = 0,
1410                 .offloads = 0,
1411         };
1412
1413         dev_info->default_txconf = (struct rte_eth_txconf) {
1414                 .tx_thresh = {
1415                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1416                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1417                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1418                 },
1419                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1420                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1421                 .offloads = 0,
1422         };
1423
1424         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1425                 .nb_max = FM10K_MAX_RX_DESC,
1426                 .nb_min = FM10K_MIN_RX_DESC,
1427                 .nb_align = FM10K_MULT_RX_DESC,
1428         };
1429
1430         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1431                 .nb_max = FM10K_MAX_TX_DESC,
1432                 .nb_min = FM10K_MIN_TX_DESC,
1433                 .nb_align = FM10K_MULT_TX_DESC,
1434                 .nb_seg_max = FM10K_TX_MAX_SEG,
1435                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1436         };
1437
1438         dev_info->speed_capa = RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_2_5G |
1439                         RTE_ETH_LINK_SPEED_10G | RTE_ETH_LINK_SPEED_25G |
1440                         RTE_ETH_LINK_SPEED_40G | RTE_ETH_LINK_SPEED_100G;
1441
1442         return 0;
1443 }
1444
1445 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1446 static const uint32_t *
1447 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1448 {
1449         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1450             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1451                 static uint32_t ptypes[] = {
1452                         /* refers to rx_desc_to_ol_flags() */
1453                         RTE_PTYPE_L2_ETHER,
1454                         RTE_PTYPE_L3_IPV4,
1455                         RTE_PTYPE_L3_IPV4_EXT,
1456                         RTE_PTYPE_L3_IPV6,
1457                         RTE_PTYPE_L3_IPV6_EXT,
1458                         RTE_PTYPE_L4_TCP,
1459                         RTE_PTYPE_L4_UDP,
1460                         RTE_PTYPE_UNKNOWN
1461                 };
1462
1463                 return ptypes;
1464         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1465                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1466                 static uint32_t ptypes_vec[] = {
1467                         /* refers to fm10k_desc_to_pktype_v() */
1468                         RTE_PTYPE_L3_IPV4,
1469                         RTE_PTYPE_L3_IPV4_EXT,
1470                         RTE_PTYPE_L3_IPV6,
1471                         RTE_PTYPE_L3_IPV6_EXT,
1472                         RTE_PTYPE_L4_TCP,
1473                         RTE_PTYPE_L4_UDP,
1474                         RTE_PTYPE_TUNNEL_GENEVE,
1475                         RTE_PTYPE_TUNNEL_NVGRE,
1476                         RTE_PTYPE_TUNNEL_VXLAN,
1477                         RTE_PTYPE_TUNNEL_GRE,
1478                         RTE_PTYPE_UNKNOWN
1479                 };
1480
1481                 return ptypes_vec;
1482         }
1483
1484         return NULL;
1485 }
1486 #else
1487 static const uint32_t *
1488 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1489 {
1490         return NULL;
1491 }
1492 #endif
1493
1494 static int
1495 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1496 {
1497         s32 result;
1498         uint16_t mac_num = 0;
1499         uint32_t vid_idx, vid_bit, mac_index;
1500         struct fm10k_hw *hw;
1501         struct fm10k_macvlan_filter_info *macvlan;
1502         struct rte_eth_dev_data *data = dev->data;
1503
1504         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1505         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1506
1507         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1508                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1509                 return -EINVAL;
1510         }
1511
1512         if (vlan_id > RTE_ETH_VLAN_ID_MAX) {
1513                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1514                 return -EINVAL;
1515         }
1516
1517         vid_idx = FM10K_VFTA_IDX(vlan_id);
1518         vid_bit = FM10K_VFTA_BIT(vlan_id);
1519         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1520         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1521                 return 0;
1522         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1523         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1524                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1525                         "in the VLAN filter table");
1526                 return -EINVAL;
1527         }
1528
1529         fm10k_mbx_lock(hw);
1530         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1531         fm10k_mbx_unlock(hw);
1532         if (result != FM10K_SUCCESS) {
1533                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1534                 return -EIO;
1535         }
1536
1537         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1538                         (result == FM10K_SUCCESS); mac_index++) {
1539                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1540                         continue;
1541                 if (mac_num > macvlan->mac_num - 1) {
1542                         PMD_INIT_LOG(ERR, "MAC address number "
1543                                         "not match");
1544                         break;
1545                 }
1546                 fm10k_mbx_lock(hw);
1547                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1548                         data->mac_addrs[mac_index].addr_bytes,
1549                         vlan_id, on, 0);
1550                 fm10k_mbx_unlock(hw);
1551                 mac_num++;
1552         }
1553         if (result != FM10K_SUCCESS) {
1554                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1555                 return -EIO;
1556         }
1557
1558         if (on) {
1559                 macvlan->vlan_num++;
1560                 macvlan->vfta[vid_idx] |= vid_bit;
1561         } else {
1562                 macvlan->vlan_num--;
1563                 macvlan->vfta[vid_idx] &= ~vid_bit;
1564         }
1565         return 0;
1566 }
1567
1568 static int
1569 fm10k_vlan_offload_set(struct rte_eth_dev *dev __rte_unused,
1570                        int mask __rte_unused)
1571 {
1572         return 0;
1573 }
1574
1575 /* Add/Remove a MAC address, and update filters to main VSI */
1576 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1577                 const u8 *mac, bool add, uint32_t pool)
1578 {
1579         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1580         struct fm10k_macvlan_filter_info *macvlan;
1581         uint32_t i, j, k;
1582
1583         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1584
1585         if (pool != MAIN_VSI_POOL_NUMBER) {
1586                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1587                         "mac to pool %u", pool);
1588                 return;
1589         }
1590         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1591                 if (!macvlan->vfta[j])
1592                         continue;
1593                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1594                         if (!(macvlan->vfta[j] & (1 << k)))
1595                                 continue;
1596                         if (i + 1 > macvlan->vlan_num) {
1597                                 PMD_INIT_LOG(ERR, "vlan number not match");
1598                                 return;
1599                         }
1600                         fm10k_mbx_lock(hw);
1601                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1602                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1603                         fm10k_mbx_unlock(hw);
1604                         i++;
1605                 }
1606         }
1607 }
1608
1609 /* Add/Remove a MAC address, and update filters to VMDQ */
1610 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1611                 const u8 *mac, bool add, uint32_t pool)
1612 {
1613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1614         struct fm10k_macvlan_filter_info *macvlan;
1615         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1616         uint32_t i;
1617
1618         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1619         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1620
1621         if (pool > macvlan->nb_queue_pools) {
1622                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1623                         " Max pool is %u",
1624                         pool, macvlan->nb_queue_pools);
1625                 return;
1626         }
1627         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1628                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1629                         continue;
1630                 fm10k_mbx_lock(hw);
1631                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1632                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1633                 fm10k_mbx_unlock(hw);
1634         }
1635 }
1636
1637 /* Add/Remove a MAC address, and update filters */
1638 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1639                 const u8 *mac, bool add, uint32_t pool)
1640 {
1641         struct fm10k_macvlan_filter_info *macvlan;
1642
1643         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1644
1645         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1646                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1647         else
1648                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1649
1650         if (add)
1651                 macvlan->mac_num++;
1652         else
1653                 macvlan->mac_num--;
1654 }
1655
1656 /* Add a MAC address, and update filters */
1657 static int
1658 fm10k_macaddr_add(struct rte_eth_dev *dev,
1659                 struct rte_ether_addr *mac_addr,
1660                 uint32_t index,
1661                 uint32_t pool)
1662 {
1663         struct fm10k_macvlan_filter_info *macvlan;
1664
1665         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1666         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1667         macvlan->mac_vmdq_id[index] = pool;
1668         return 0;
1669 }
1670
1671 /* Remove a MAC address, and update filters */
1672 static void
1673 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1674 {
1675         struct rte_eth_dev_data *data = dev->data;
1676         struct fm10k_macvlan_filter_info *macvlan;
1677
1678         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1679         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1680                         FALSE, macvlan->mac_vmdq_id[index]);
1681         macvlan->mac_vmdq_id[index] = 0;
1682 }
1683
1684 static inline int
1685 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1686 {
1687         if ((request < min) || (request > max) || ((request % mult) != 0))
1688                 return -1;
1689         else
1690                 return 0;
1691 }
1692
1693
1694 static inline int
1695 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1696 {
1697         if ((request < min) || (request > max) || ((div % request) != 0))
1698                 return -1;
1699         else
1700                 return 0;
1701 }
1702
1703 static inline int
1704 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1705 {
1706         uint16_t rx_free_thresh;
1707
1708         if (conf->rx_free_thresh == 0)
1709                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1710         else
1711                 rx_free_thresh = conf->rx_free_thresh;
1712
1713         /* make sure the requested threshold satisfies the constraints */
1714         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1715                         FM10K_RX_FREE_THRESH_MAX(q),
1716                         FM10K_RX_FREE_THRESH_DIV(q),
1717                         rx_free_thresh)) {
1718                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1719                         "less than or equal to %u, "
1720                         "greater than or equal to %u, "
1721                         "and a divisor of %u",
1722                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1723                         FM10K_RX_FREE_THRESH_MIN(q),
1724                         FM10K_RX_FREE_THRESH_DIV(q));
1725                 return -EINVAL;
1726         }
1727
1728         q->alloc_thresh = rx_free_thresh;
1729         q->drop_en = conf->rx_drop_en;
1730         q->rx_deferred_start = conf->rx_deferred_start;
1731
1732         return 0;
1733 }
1734
1735 /*
1736  * Hardware requires specific alignment for Rx packet buffers. At
1737  * least one of the following two conditions must be satisfied.
1738  *  1. Address is 512B aligned
1739  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1740  *
1741  * As such, the driver may need to adjust the DMA address within the
1742  * buffer by up to 512B.
1743  *
1744  * return 1 if the element size is valid, otherwise return 0.
1745  */
1746 static int
1747 mempool_element_size_valid(struct rte_mempool *mp)
1748 {
1749         uint32_t min_size;
1750
1751         /* elt_size includes mbuf header and headroom */
1752         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1753                         RTE_PKTMBUF_HEADROOM;
1754
1755         /* account for up to 512B of alignment */
1756         min_size -= FM10K_RX_DATABUF_ALIGN;
1757
1758         /* sanity check for overflow */
1759         if (min_size > mp->elt_size)
1760                 return 0;
1761
1762         /* size is valid */
1763         return 1;
1764 }
1765
1766 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1767 {
1768         RTE_SET_USED(dev);
1769
1770         return (uint64_t)(RTE_ETH_RX_OFFLOAD_SCATTER);
1771 }
1772
1773 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1774 {
1775         RTE_SET_USED(dev);
1776
1777         return  (uint64_t)(RTE_ETH_RX_OFFLOAD_VLAN_STRIP  |
1778                            RTE_ETH_RX_OFFLOAD_VLAN_FILTER |
1779                            RTE_ETH_RX_OFFLOAD_IPV4_CKSUM  |
1780                            RTE_ETH_RX_OFFLOAD_UDP_CKSUM   |
1781                            RTE_ETH_RX_OFFLOAD_TCP_CKSUM   |
1782                            RTE_ETH_RX_OFFLOAD_HEADER_SPLIT |
1783                            RTE_ETH_RX_OFFLOAD_RSS_HASH);
1784 }
1785
1786 static int
1787 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1788         uint16_t nb_desc, unsigned int socket_id,
1789         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1790 {
1791         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1792         struct fm10k_dev_info *dev_info =
1793                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1794         struct fm10k_rx_queue *q;
1795         const struct rte_memzone *mz;
1796         uint64_t offloads;
1797
1798         PMD_INIT_FUNC_TRACE();
1799
1800         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1801
1802         /* make sure the mempool element size can account for alignment. */
1803         if (!mempool_element_size_valid(mp)) {
1804                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1805                 return -EINVAL;
1806         }
1807
1808         /* make sure a valid number of descriptors have been requested */
1809         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1810                                 FM10K_MULT_RX_DESC, nb_desc)) {
1811                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1812                         "less than or equal to %"PRIu32", "
1813                         "greater than or equal to %u, "
1814                         "and a multiple of %u",
1815                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1816                         FM10K_MULT_RX_DESC);
1817                 return -EINVAL;
1818         }
1819
1820         /*
1821          * if this queue existed already, free the associated memory. The
1822          * queue cannot be reused in case we need to allocate memory on
1823          * different socket than was previously used.
1824          */
1825         if (dev->data->rx_queues[queue_id] != NULL) {
1826                 rx_queue_free(dev->data->rx_queues[queue_id]);
1827                 dev->data->rx_queues[queue_id] = NULL;
1828         }
1829
1830         /* allocate memory for the queue structure */
1831         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1832                                 socket_id);
1833         if (q == NULL) {
1834                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1835                 return -ENOMEM;
1836         }
1837
1838         /* setup queue */
1839         q->mp = mp;
1840         q->nb_desc = nb_desc;
1841         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1842         q->port_id = dev->data->port_id;
1843         q->queue_id = queue_id;
1844         q->tail_ptr = (volatile uint32_t *)
1845                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1846         q->offloads = offloads;
1847         if (handle_rxconf(q, conf)) {
1848                 rte_free(q);
1849                 return -EINVAL;
1850         }
1851         /* allocate memory for the software ring */
1852         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1853                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1854                         RTE_CACHE_LINE_SIZE, socket_id);
1855         if (q->sw_ring == NULL) {
1856                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1857                 rte_free(q);
1858                 return -ENOMEM;
1859         }
1860
1861         /*
1862          * allocate memory for the hardware descriptor ring. A memzone large
1863          * enough to hold the maximum ring size is requested to allow for
1864          * resizing in later calls to the queue setup function.
1865          */
1866         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1867                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1868                                       socket_id);
1869         if (mz == NULL) {
1870                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1871                 rte_free(q->sw_ring);
1872                 rte_free(q);
1873                 return -ENOMEM;
1874         }
1875         q->hw_ring = mz->addr;
1876         q->hw_ring_phys_addr = mz->iova;
1877
1878         /* Check if number of descs satisfied Vector requirement */
1879         if (!rte_is_power_of_2(nb_desc)) {
1880                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1881                                     "preconditions - canceling the feature for "
1882                                     "the whole port[%d]",
1883                              q->queue_id, q->port_id);
1884                 dev_info->rx_vec_allowed = false;
1885         } else
1886                 fm10k_rxq_vec_setup(q);
1887
1888         dev->data->rx_queues[queue_id] = q;
1889         return 0;
1890 }
1891
1892 static void
1893 fm10k_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
1894 {
1895         PMD_INIT_FUNC_TRACE();
1896
1897         rx_queue_free(dev->data->rx_queues[qid]);
1898 }
1899
1900 static inline int
1901 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1902 {
1903         uint16_t tx_free_thresh;
1904         uint16_t tx_rs_thresh;
1905
1906         /* constraint MACROs require that tx_free_thresh is configured
1907          * before tx_rs_thresh */
1908         if (conf->tx_free_thresh == 0)
1909                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1910         else
1911                 tx_free_thresh = conf->tx_free_thresh;
1912
1913         /* make sure the requested threshold satisfies the constraints */
1914         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1915                         FM10K_TX_FREE_THRESH_MAX(q),
1916                         FM10K_TX_FREE_THRESH_DIV(q),
1917                         tx_free_thresh)) {
1918                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1919                         "less than or equal to %u, "
1920                         "greater than or equal to %u, "
1921                         "and a divisor of %u",
1922                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1923                         FM10K_TX_FREE_THRESH_MIN(q),
1924                         FM10K_TX_FREE_THRESH_DIV(q));
1925                 return -EINVAL;
1926         }
1927
1928         q->free_thresh = tx_free_thresh;
1929
1930         if (conf->tx_rs_thresh == 0)
1931                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1932         else
1933                 tx_rs_thresh = conf->tx_rs_thresh;
1934
1935         q->tx_deferred_start = conf->tx_deferred_start;
1936
1937         /* make sure the requested threshold satisfies the constraints */
1938         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1939                         FM10K_TX_RS_THRESH_MAX(q),
1940                         FM10K_TX_RS_THRESH_DIV(q),
1941                         tx_rs_thresh)) {
1942                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1943                         "less than or equal to %u, "
1944                         "greater than or equal to %u, "
1945                         "and a divisor of %u",
1946                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1947                         FM10K_TX_RS_THRESH_MIN(q),
1948                         FM10K_TX_RS_THRESH_DIV(q));
1949                 return -EINVAL;
1950         }
1951
1952         q->rs_thresh = tx_rs_thresh;
1953
1954         return 0;
1955 }
1956
1957 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1958 {
1959         RTE_SET_USED(dev);
1960
1961         return 0;
1962 }
1963
1964 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1965 {
1966         RTE_SET_USED(dev);
1967
1968         return (uint64_t)(RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
1969                           RTE_ETH_TX_OFFLOAD_MULTI_SEGS  |
1970                           RTE_ETH_TX_OFFLOAD_IPV4_CKSUM  |
1971                           RTE_ETH_TX_OFFLOAD_UDP_CKSUM   |
1972                           RTE_ETH_TX_OFFLOAD_TCP_CKSUM   |
1973                           RTE_ETH_TX_OFFLOAD_TCP_TSO);
1974 }
1975
1976 static int
1977 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1978         uint16_t nb_desc, unsigned int socket_id,
1979         const struct rte_eth_txconf *conf)
1980 {
1981         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1982         struct fm10k_tx_queue *q;
1983         const struct rte_memzone *mz;
1984         uint64_t offloads;
1985
1986         PMD_INIT_FUNC_TRACE();
1987
1988         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
1989
1990         /* make sure a valid number of descriptors have been requested */
1991         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1992                                 FM10K_MULT_TX_DESC, nb_desc)) {
1993                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1994                         "less than or equal to %"PRIu32", "
1995                         "greater than or equal to %u, "
1996                         "and a multiple of %u",
1997                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1998                         FM10K_MULT_TX_DESC);
1999                 return -EINVAL;
2000         }
2001
2002         /*
2003          * if this queue existed already, free the associated memory. The
2004          * queue cannot be reused in case we need to allocate memory on
2005          * different socket than was previously used.
2006          */
2007         if (dev->data->tx_queues[queue_id] != NULL) {
2008                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2009
2010                 tx_queue_free(txq);
2011                 dev->data->tx_queues[queue_id] = NULL;
2012         }
2013
2014         /* allocate memory for the queue structure */
2015         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2016                                 socket_id);
2017         if (q == NULL) {
2018                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2019                 return -ENOMEM;
2020         }
2021
2022         /* setup queue */
2023         q->nb_desc = nb_desc;
2024         q->port_id = dev->data->port_id;
2025         q->queue_id = queue_id;
2026         q->offloads = offloads;
2027         q->ops = &def_txq_ops;
2028         q->tail_ptr = (volatile uint32_t *)
2029                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2030         if (handle_txconf(q, conf)) {
2031                 rte_free(q);
2032                 return -EINVAL;
2033         }
2034
2035         /* allocate memory for the software ring */
2036         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2037                                         nb_desc * sizeof(struct rte_mbuf *),
2038                                         RTE_CACHE_LINE_SIZE, socket_id);
2039         if (q->sw_ring == NULL) {
2040                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2041                 rte_free(q);
2042                 return -ENOMEM;
2043         }
2044
2045         /*
2046          * allocate memory for the hardware descriptor ring. A memzone large
2047          * enough to hold the maximum ring size is requested to allow for
2048          * resizing in later calls to the queue setup function.
2049          */
2050         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2051                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2052                                       socket_id);
2053         if (mz == NULL) {
2054                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2055                 rte_free(q->sw_ring);
2056                 rte_free(q);
2057                 return -ENOMEM;
2058         }
2059         q->hw_ring = mz->addr;
2060         q->hw_ring_phys_addr = mz->iova;
2061
2062         /*
2063          * allocate memory for the RS bit tracker. Enough slots to hold the
2064          * descriptor index for each RS bit needing to be set are required.
2065          */
2066         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2067                                 ((nb_desc + 1) / q->rs_thresh) *
2068                                 sizeof(uint16_t),
2069                                 RTE_CACHE_LINE_SIZE, socket_id);
2070         if (q->rs_tracker.list == NULL) {
2071                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2072                 rte_free(q->sw_ring);
2073                 rte_free(q);
2074                 return -ENOMEM;
2075         }
2076
2077         dev->data->tx_queues[queue_id] = q;
2078         return 0;
2079 }
2080
2081 static void
2082 fm10k_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
2083 {
2084         struct fm10k_tx_queue *q = dev->data->tx_queues[qid];
2085         PMD_INIT_FUNC_TRACE();
2086
2087         tx_queue_free(q);
2088 }
2089
2090 static int
2091 fm10k_reta_update(struct rte_eth_dev *dev,
2092                         struct rte_eth_rss_reta_entry64 *reta_conf,
2093                         uint16_t reta_size)
2094 {
2095         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2096         uint16_t i, j, idx, shift;
2097         uint8_t mask;
2098         uint32_t reta;
2099
2100         PMD_INIT_FUNC_TRACE();
2101
2102         if (reta_size > FM10K_MAX_RSS_INDICES) {
2103                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2104                         "(%d) doesn't match the number hardware can supported "
2105                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2106                 return -EINVAL;
2107         }
2108
2109         /*
2110          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2111          * 128-entries in 32 registers
2112          */
2113         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2114                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2115                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2116                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2117                                 BIT_MASK_PER_UINT32);
2118                 if (mask == 0)
2119                         continue;
2120
2121                 reta = 0;
2122                 if (mask != BIT_MASK_PER_UINT32)
2123                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2124
2125                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2126                         if (mask & (0x1 << j)) {
2127                                 if (mask != 0xF)
2128                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2129                                 reta |= reta_conf[idx].reta[shift + j] <<
2130                                                 (CHAR_BIT * j);
2131                         }
2132                 }
2133                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2134         }
2135
2136         return 0;
2137 }
2138
2139 static int
2140 fm10k_reta_query(struct rte_eth_dev *dev,
2141                         struct rte_eth_rss_reta_entry64 *reta_conf,
2142                         uint16_t reta_size)
2143 {
2144         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2145         uint16_t i, j, idx, shift;
2146         uint8_t mask;
2147         uint32_t reta;
2148
2149         PMD_INIT_FUNC_TRACE();
2150
2151         if (reta_size < FM10K_MAX_RSS_INDICES) {
2152                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2153                         "(%d) doesn't match the number hardware can supported "
2154                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2155                 return -EINVAL;
2156         }
2157
2158         /*
2159          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2160          * 128-entries in 32 registers
2161          */
2162         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2163                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2164                 shift = i % RTE_ETH_RETA_GROUP_SIZE;
2165                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2166                                 BIT_MASK_PER_UINT32);
2167                 if (mask == 0)
2168                         continue;
2169
2170                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2171                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2172                         if (mask & (0x1 << j))
2173                                 reta_conf[idx].reta[shift + j] = ((reta >>
2174                                         CHAR_BIT * j) & UINT8_MAX);
2175                 }
2176         }
2177
2178         return 0;
2179 }
2180
2181 static int
2182 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2183         struct rte_eth_rss_conf *rss_conf)
2184 {
2185         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2186         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2187         uint32_t mrqc;
2188         uint64_t hf = rss_conf->rss_hf;
2189         int i;
2190
2191         PMD_INIT_FUNC_TRACE();
2192
2193         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2194                                 FM10K_RSSRK_ENTRIES_PER_REG))
2195                 return -EINVAL;
2196
2197         if (hf == 0)
2198                 return -EINVAL;
2199
2200         mrqc = 0;
2201         mrqc |= (hf & RTE_ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2202         mrqc |= (hf & RTE_ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2203         mrqc |= (hf & RTE_ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2204         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2205         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2206         mrqc |= (hf & RTE_ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2207         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2208         mrqc |= (hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2209         mrqc |= (hf & RTE_ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2210
2211         /* If the mapping doesn't fit any supported, return */
2212         if (mrqc == 0)
2213                 return -EINVAL;
2214
2215         if (key != NULL)
2216                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2217                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2218
2219         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2220
2221         return 0;
2222 }
2223
2224 static int
2225 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2226         struct rte_eth_rss_conf *rss_conf)
2227 {
2228         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2229         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2230         uint32_t mrqc;
2231         uint64_t hf;
2232         int i;
2233
2234         PMD_INIT_FUNC_TRACE();
2235
2236         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2237                                 FM10K_RSSRK_ENTRIES_PER_REG))
2238                 return -EINVAL;
2239
2240         if (key != NULL)
2241                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2242                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2243
2244         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2245         hf = 0;
2246         hf |= (mrqc & FM10K_MRQC_IPV4)     ? RTE_ETH_RSS_IPV4              : 0;
2247         hf |= (mrqc & FM10K_MRQC_IPV6)     ? RTE_ETH_RSS_IPV6              : 0;
2248         hf |= (mrqc & FM10K_MRQC_IPV6)     ? RTE_ETH_RSS_IPV6_EX           : 0;
2249         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? RTE_ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2250         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? RTE_ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2251         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? RTE_ETH_RSS_IPV6_TCP_EX       : 0;
2252         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? RTE_ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2253         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? RTE_ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2254         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? RTE_ETH_RSS_IPV6_UDP_EX       : 0;
2255
2256         rss_conf->rss_hf = hf;
2257
2258         return 0;
2259 }
2260
2261 static void
2262 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2263 {
2264         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2265         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2266
2267         /* Bind all local non-queue interrupt to vector 0 */
2268         int_map |= FM10K_MISC_VEC_ID;
2269
2270         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2271         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2272         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2273         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2274         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2275         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2276
2277         /* Enable misc causes */
2278         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2279                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2280                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2281                                 FM10K_EIMR_ENABLE(MAILBOX) |
2282                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2283                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2284                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2285                                 FM10K_EIMR_ENABLE(VFLR));
2286
2287         /* Enable ITR 0 */
2288         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2289                                         FM10K_ITR_MASK_CLEAR);
2290         FM10K_WRITE_FLUSH(hw);
2291 }
2292
2293 static void
2294 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2295 {
2296         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2297         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2298
2299         int_map |= FM10K_MISC_VEC_ID;
2300
2301         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2302         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2303         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2304         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2305         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2306         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2307
2308         /* Disable misc causes */
2309         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2310                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2311                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2312                                 FM10K_EIMR_DISABLE(MAILBOX) |
2313                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2314                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2315                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2316                                 FM10K_EIMR_DISABLE(VFLR));
2317
2318         /* Disable ITR 0 */
2319         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2320         FM10K_WRITE_FLUSH(hw);
2321 }
2322
2323 static void
2324 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2325 {
2326         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2327         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2328
2329         /* Bind all local non-queue interrupt to vector 0 */
2330         int_map |= FM10K_MISC_VEC_ID;
2331
2332         /* Only INT 0 available, other 15 are reserved. */
2333         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2334
2335         /* Enable ITR 0 */
2336         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2337                                         FM10K_ITR_MASK_CLEAR);
2338         FM10K_WRITE_FLUSH(hw);
2339 }
2340
2341 static void
2342 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2343 {
2344         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2345         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2346
2347         int_map |= FM10K_MISC_VEC_ID;
2348
2349         /* Only INT 0 available, other 15 are reserved. */
2350         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2351
2352         /* Disable ITR 0 */
2353         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2354         FM10K_WRITE_FLUSH(hw);
2355 }
2356
2357 static int
2358 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2359 {
2360         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2361         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2362
2363         /* Enable ITR */
2364         if (hw->mac.type == fm10k_mac_pf)
2365                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2366                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2367         else
2368                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2369                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2370         rte_intr_ack(pdev->intr_handle);
2371         return 0;
2372 }
2373
2374 static int
2375 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2376 {
2377         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2378         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2379
2380         /* Disable ITR */
2381         if (hw->mac.type == fm10k_mac_pf)
2382                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2383                         FM10K_ITR_MASK_SET);
2384         else
2385                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2386                         FM10K_ITR_MASK_SET);
2387         return 0;
2388 }
2389
2390 static int
2391 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2392 {
2393         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2394         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2395         struct rte_intr_handle *intr_handle = pdev->intr_handle;
2396         uint32_t intr_vector, vec;
2397         uint16_t queue_id;
2398         int result = 0;
2399
2400         /* fm10k needs one separate interrupt for mailbox,
2401          * so only drivers which support multiple interrupt vectors
2402          * e.g. vfio-pci can work for fm10k interrupt mode
2403          */
2404         if (!rte_intr_cap_multiple(intr_handle) ||
2405                         dev->data->dev_conf.intr_conf.rxq == 0)
2406                 return result;
2407
2408         intr_vector = dev->data->nb_rx_queues;
2409
2410         /* disable interrupt first */
2411         rte_intr_disable(intr_handle);
2412         if (hw->mac.type == fm10k_mac_pf)
2413                 fm10k_dev_disable_intr_pf(dev);
2414         else
2415                 fm10k_dev_disable_intr_vf(dev);
2416
2417         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2418                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2419                 result = -EIO;
2420         }
2421
2422         if (rte_intr_dp_is_en(intr_handle) && !result) {
2423                 if (!rte_intr_vec_list_alloc(intr_handle, "intr_vec",
2424                                                    dev->data->nb_rx_queues)) {
2425                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2426                                         queue_id < dev->data->nb_rx_queues;
2427                                         queue_id++) {
2428                                 rte_intr_vec_list_index_set(intr_handle,
2429                                                                 queue_id, vec);
2430                                 int nb_efd =
2431                                         rte_intr_nb_efd_get(intr_handle);
2432                                 if (vec < (uint32_t)nb_efd - 1 +
2433                                                         FM10K_RX_VEC_START)
2434                                         vec++;
2435                         }
2436                 } else {
2437                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2438                                 " intr_vec", dev->data->nb_rx_queues);
2439                         rte_intr_efd_disable(intr_handle);
2440                         result = -ENOMEM;
2441                 }
2442         }
2443
2444         if (hw->mac.type == fm10k_mac_pf)
2445                 fm10k_dev_enable_intr_pf(dev);
2446         else
2447                 fm10k_dev_enable_intr_vf(dev);
2448         rte_intr_enable(intr_handle);
2449         hw->mac.ops.update_int_moderator(hw);
2450         return result;
2451 }
2452
2453 static int
2454 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2455 {
2456         struct fm10k_fault fault;
2457         int err;
2458         const char *estr = "Unknown error";
2459
2460         /* Process PCA fault */
2461         if (eicr & FM10K_EICR_PCA_FAULT) {
2462                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2463                 if (err)
2464                         goto error;
2465                 switch (fault.type) {
2466                 case PCA_NO_FAULT:
2467                         estr = "PCA_NO_FAULT"; break;
2468                 case PCA_UNMAPPED_ADDR:
2469                         estr = "PCA_UNMAPPED_ADDR"; break;
2470                 case PCA_BAD_QACCESS_PF:
2471                         estr = "PCA_BAD_QACCESS_PF"; break;
2472                 case PCA_BAD_QACCESS_VF:
2473                         estr = "PCA_BAD_QACCESS_VF"; break;
2474                 case PCA_MALICIOUS_REQ:
2475                         estr = "PCA_MALICIOUS_REQ"; break;
2476                 case PCA_POISONED_TLP:
2477                         estr = "PCA_POISONED_TLP"; break;
2478                 case PCA_TLP_ABORT:
2479                         estr = "PCA_TLP_ABORT"; break;
2480                 default:
2481                         goto error;
2482                 }
2483                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2484                         estr, fault.func ? "VF" : "PF", fault.func,
2485                         fault.address, fault.specinfo);
2486         }
2487
2488         /* Process THI fault */
2489         if (eicr & FM10K_EICR_THI_FAULT) {
2490                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2491                 if (err)
2492                         goto error;
2493                 switch (fault.type) {
2494                 case THI_NO_FAULT:
2495                         estr = "THI_NO_FAULT"; break;
2496                 case THI_MAL_DIS_Q_FAULT:
2497                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2498                 default:
2499                         goto error;
2500                 }
2501                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2502                         estr, fault.func ? "VF" : "PF", fault.func,
2503                         fault.address, fault.specinfo);
2504         }
2505
2506         /* Process FUM fault */
2507         if (eicr & FM10K_EICR_FUM_FAULT) {
2508                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2509                 if (err)
2510                         goto error;
2511                 switch (fault.type) {
2512                 case FUM_NO_FAULT:
2513                         estr = "FUM_NO_FAULT"; break;
2514                 case FUM_UNMAPPED_ADDR:
2515                         estr = "FUM_UNMAPPED_ADDR"; break;
2516                 case FUM_POISONED_TLP:
2517                         estr = "FUM_POISONED_TLP"; break;
2518                 case FUM_BAD_VF_QACCESS:
2519                         estr = "FUM_BAD_VF_QACCESS"; break;
2520                 case FUM_ADD_DECODE_ERR:
2521                         estr = "FUM_ADD_DECODE_ERR"; break;
2522                 case FUM_RO_ERROR:
2523                         estr = "FUM_RO_ERROR"; break;
2524                 case FUM_QPRC_CRC_ERROR:
2525                         estr = "FUM_QPRC_CRC_ERROR"; break;
2526                 case FUM_CSR_TIMEOUT:
2527                         estr = "FUM_CSR_TIMEOUT"; break;
2528                 case FUM_INVALID_TYPE:
2529                         estr = "FUM_INVALID_TYPE"; break;
2530                 case FUM_INVALID_LENGTH:
2531                         estr = "FUM_INVALID_LENGTH"; break;
2532                 case FUM_INVALID_BE:
2533                         estr = "FUM_INVALID_BE"; break;
2534                 case FUM_INVALID_ALIGN:
2535                         estr = "FUM_INVALID_ALIGN"; break;
2536                 default:
2537                         goto error;
2538                 }
2539                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2540                         estr, fault.func ? "VF" : "PF", fault.func,
2541                         fault.address, fault.specinfo);
2542         }
2543
2544         return 0;
2545 error:
2546         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2547         return err;
2548 }
2549
2550 /**
2551  * PF interrupt handler triggered by NIC for handling specific interrupt.
2552  *
2553  * @param handle
2554  *  Pointer to interrupt handle.
2555  * @param param
2556  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2557  *
2558  * @return
2559  *  void
2560  */
2561 static void
2562 fm10k_dev_interrupt_handler_pf(void *param)
2563 {
2564         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2565         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2566         uint32_t cause, status;
2567         struct fm10k_dev_info *dev_info =
2568                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2569         int status_mbx;
2570         s32 err;
2571
2572         if (hw->mac.type != fm10k_mac_pf)
2573                 return;
2574
2575         cause = FM10K_READ_REG(hw, FM10K_EICR);
2576
2577         /* Handle PCI fault cases */
2578         if (cause & FM10K_EICR_FAULT_MASK) {
2579                 PMD_INIT_LOG(ERR, "INT: find fault!");
2580                 fm10k_dev_handle_fault(hw, cause);
2581         }
2582
2583         /* Handle switch up/down */
2584         if (cause & FM10K_EICR_SWITCHNOTREADY)
2585                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2586
2587         if (cause & FM10K_EICR_SWITCHREADY) {
2588                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2589                 if (dev_info->sm_down == 1) {
2590                         fm10k_mbx_lock(hw);
2591
2592                         /* For recreating logical ports */
2593                         status_mbx = hw->mac.ops.update_lport_state(hw,
2594                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2595                         if (status_mbx == FM10K_SUCCESS)
2596                                 PMD_INIT_LOG(INFO,
2597                                         "INT: Recreated Logical port");
2598                         else
2599                                 PMD_INIT_LOG(INFO,
2600                                         "INT: Logical ports weren't recreated");
2601
2602                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2603                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2604                         if (status_mbx != FM10K_SUCCESS)
2605                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2606
2607                         fm10k_mbx_unlock(hw);
2608
2609                         /* first clear the internal SW recording structure */
2610                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2611                                                 RTE_ETH_MQ_RX_VMDQ_FLAG))
2612                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2613                                         false);
2614
2615                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2616                                         MAIN_VSI_POOL_NUMBER);
2617
2618                         /*
2619                          * Add default mac address and vlan for the logical
2620                          * ports that have been created, leave to the
2621                          * application to fully recover Rx filtering.
2622                          */
2623                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2624                                         MAIN_VSI_POOL_NUMBER);
2625
2626                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2627                                                 RTE_ETH_MQ_RX_VMDQ_FLAG))
2628                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2629                                         true);
2630
2631                         dev_info->sm_down = 0;
2632                         rte_eth_dev_callback_process(dev,
2633                                         RTE_ETH_EVENT_INTR_LSC,
2634                                         NULL);
2635                 }
2636         }
2637
2638         /* Handle mailbox message */
2639         fm10k_mbx_lock(hw);
2640         err = hw->mbx.ops.process(hw, &hw->mbx);
2641         fm10k_mbx_unlock(hw);
2642
2643         if (err == FM10K_ERR_RESET_REQUESTED) {
2644                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2645                 dev_info->sm_down = 1;
2646                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2647         }
2648
2649         /* Handle SRAM error */
2650         if (cause & FM10K_EICR_SRAMERROR) {
2651                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2652
2653                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2654                 /* Write to clear pending bits */
2655                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2656
2657                 /* Todo: print out error message after shared code  updates */
2658         }
2659
2660         /* Clear these 3 events if having any */
2661         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2662                  FM10K_EICR_SWITCHREADY;
2663         if (cause)
2664                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2665
2666         /* Re-enable interrupt from device side */
2667         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2668                                         FM10K_ITR_MASK_CLEAR);
2669         /* Re-enable interrupt from host side */
2670         rte_intr_ack(dev->intr_handle);
2671 }
2672
2673 /**
2674  * VF interrupt handler triggered by NIC for handling specific interrupt.
2675  *
2676  * @param handle
2677  *  Pointer to interrupt handle.
2678  * @param param
2679  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2680  *
2681  * @return
2682  *  void
2683  */
2684 static void
2685 fm10k_dev_interrupt_handler_vf(void *param)
2686 {
2687         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2688         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2689         struct fm10k_mbx_info *mbx = &hw->mbx;
2690         struct fm10k_dev_info *dev_info =
2691                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2692         const enum fm10k_mbx_state state = mbx->state;
2693         int status_mbx;
2694
2695         if (hw->mac.type != fm10k_mac_vf)
2696                 return;
2697
2698         /* Handle mailbox message if lock is acquired */
2699         fm10k_mbx_lock(hw);
2700         hw->mbx.ops.process(hw, &hw->mbx);
2701         fm10k_mbx_unlock(hw);
2702
2703         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2704                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2705
2706                 fm10k_mbx_lock(hw);
2707                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2708                                 MAX_LPORT_NUM, 1);
2709                 fm10k_mbx_unlock(hw);
2710
2711                 /* Setting reset flag */
2712                 dev_info->sm_down = 1;
2713                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2714         }
2715
2716         if (dev_info->sm_down == 1 &&
2717                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2718                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2719                 fm10k_mbx_lock(hw);
2720                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2721                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2722                 if (status_mbx != FM10K_SUCCESS)
2723                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2724                 fm10k_mbx_unlock(hw);
2725
2726                 /* first clear the internal SW recording structure */
2727                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2728                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2729                                 MAIN_VSI_POOL_NUMBER);
2730
2731                 /*
2732                  * Add default mac address and vlan for the logical ports that
2733                  * have been created, leave to the application to fully recover
2734                  * Rx filtering.
2735                  */
2736                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2737                                 MAIN_VSI_POOL_NUMBER);
2738                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2739
2740                 dev_info->sm_down = 0;
2741                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2742         }
2743
2744         /* Re-enable interrupt from device side */
2745         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2746                                         FM10K_ITR_MASK_CLEAR);
2747         /* Re-enable interrupt from host side */
2748         rte_intr_ack(dev->intr_handle);
2749 }
2750
2751 /* Mailbox message handler in VF */
2752 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2753         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2754         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2755         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2756         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2757 };
2758
2759 static int
2760 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2761 {
2762         int err = 0;
2763
2764         /* Initialize mailbox lock */
2765         fm10k_mbx_initlock(hw);
2766
2767         /* Replace default message handler with new ones */
2768         if (hw->mac.type == fm10k_mac_vf)
2769                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2770
2771         if (err) {
2772                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2773                                 err);
2774                 return err;
2775         }
2776         /* Connect to SM for PF device or PF for VF device */
2777         return hw->mbx.ops.connect(hw, &hw->mbx);
2778 }
2779
2780 static void
2781 fm10k_close_mbx_service(struct fm10k_hw *hw)
2782 {
2783         /* Disconnect from SM for PF device or PF for VF device */
2784         hw->mbx.ops.disconnect(hw, &hw->mbx);
2785 }
2786
2787 static int
2788 fm10k_dev_close(struct rte_eth_dev *dev)
2789 {
2790         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2791         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2792         struct rte_intr_handle *intr_handle = pdev->intr_handle;
2793         int ret;
2794
2795         PMD_INIT_FUNC_TRACE();
2796         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2797                 return 0;
2798
2799         fm10k_mbx_lock(hw);
2800         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2801                 MAX_LPORT_NUM, false);
2802         fm10k_mbx_unlock(hw);
2803
2804         /* allow 100ms for device to quiesce */
2805         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2806
2807         /* Stop mailbox service first */
2808         fm10k_close_mbx_service(hw);
2809
2810         ret = fm10k_dev_stop(dev);
2811
2812         fm10k_dev_queue_release(dev);
2813         fm10k_stop_hw(hw);
2814
2815         /* disable uio/vfio intr */
2816         rte_intr_disable(intr_handle);
2817
2818         /*PF/VF has different interrupt handling mechanism */
2819         if (hw->mac.type == fm10k_mac_pf) {
2820                 /* disable interrupt */
2821                 fm10k_dev_disable_intr_pf(dev);
2822
2823                 /* unregister callback func to eal lib */
2824                 rte_intr_callback_unregister(intr_handle,
2825                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2826         } else {
2827                 /* disable interrupt */
2828                 fm10k_dev_disable_intr_vf(dev);
2829
2830                 rte_intr_callback_unregister(intr_handle,
2831                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2832         }
2833
2834         return ret;
2835 }
2836
2837 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2838         .dev_configure          = fm10k_dev_configure,
2839         .dev_start              = fm10k_dev_start,
2840         .dev_stop               = fm10k_dev_stop,
2841         .dev_close              = fm10k_dev_close,
2842         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2843         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2844         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2845         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2846         .stats_get              = fm10k_stats_get,
2847         .xstats_get             = fm10k_xstats_get,
2848         .xstats_get_names       = fm10k_xstats_get_names,
2849         .stats_reset            = fm10k_stats_reset,
2850         .xstats_reset           = fm10k_stats_reset,
2851         .link_update            = fm10k_link_update,
2852         .dev_infos_get          = fm10k_dev_infos_get,
2853         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2854         .vlan_filter_set        = fm10k_vlan_filter_set,
2855         .vlan_offload_set       = fm10k_vlan_offload_set,
2856         .mac_addr_add           = fm10k_macaddr_add,
2857         .mac_addr_remove        = fm10k_macaddr_remove,
2858         .rx_queue_start         = fm10k_dev_rx_queue_start,
2859         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2860         .tx_queue_start         = fm10k_dev_tx_queue_start,
2861         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2862         .rx_queue_setup         = fm10k_rx_queue_setup,
2863         .rx_queue_release       = fm10k_rx_queue_release,
2864         .tx_queue_setup         = fm10k_tx_queue_setup,
2865         .tx_queue_release       = fm10k_tx_queue_release,
2866         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2867         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2868         .reta_update            = fm10k_reta_update,
2869         .reta_query             = fm10k_reta_query,
2870         .rss_hash_update        = fm10k_rss_hash_update,
2871         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2872 };
2873
2874 static int ftag_check_handler(__rte_unused const char *key,
2875                 const char *value, __rte_unused void *opaque)
2876 {
2877         if (strcmp(value, "1"))
2878                 return -1;
2879
2880         return 0;
2881 }
2882
2883 static int
2884 fm10k_check_ftag(struct rte_devargs *devargs)
2885 {
2886         struct rte_kvargs *kvlist;
2887         const char *ftag_key = "enable_ftag";
2888
2889         if (devargs == NULL)
2890                 return 0;
2891
2892         kvlist = rte_kvargs_parse(devargs->args, NULL);
2893         if (kvlist == NULL)
2894                 return 0;
2895
2896         if (!rte_kvargs_count(kvlist, ftag_key)) {
2897                 rte_kvargs_free(kvlist);
2898                 return 0;
2899         }
2900         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2901         if (rte_kvargs_process(kvlist, ftag_key,
2902                                 ftag_check_handler, NULL) < 0) {
2903                 rte_kvargs_free(kvlist);
2904                 return 0;
2905         }
2906         rte_kvargs_free(kvlist);
2907
2908         return 1;
2909 }
2910
2911 static uint16_t
2912 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2913                     uint16_t nb_pkts)
2914 {
2915         uint16_t nb_tx = 0;
2916         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2917
2918         while (nb_pkts) {
2919                 uint16_t ret, num;
2920
2921                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2922                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2923                                                  num);
2924                 nb_tx += ret;
2925                 nb_pkts -= ret;
2926                 if (ret < num)
2927                         break;
2928         }
2929
2930         return nb_tx;
2931 }
2932
2933 static void __rte_cold
2934 fm10k_set_tx_function(struct rte_eth_dev *dev)
2935 {
2936         struct fm10k_tx_queue *txq;
2937         int i;
2938         int use_sse = 1;
2939         uint16_t tx_ftag_en = 0;
2940
2941         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2942                 /* primary process has set the ftag flag and offloads */
2943                 txq = dev->data->tx_queues[0];
2944                 if (fm10k_tx_vec_condition_check(txq) ||
2945                                 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) {
2946                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2947                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2948                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2949                 } else {
2950                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2951                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2952                         dev->tx_pkt_prepare = NULL;
2953                 }
2954                 return;
2955         }
2956
2957         if (fm10k_check_ftag(dev->device->devargs))
2958                 tx_ftag_en = 1;
2959
2960         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2961                 txq = dev->data->tx_queues[i];
2962                 txq->tx_ftag_en = tx_ftag_en;
2963                 /* Check if Vector Tx is satisfied */
2964                 if (fm10k_tx_vec_condition_check(txq) ||
2965                                 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)
2966                         use_sse = 0;
2967         }
2968
2969         if (use_sse) {
2970                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2971                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2972                         txq = dev->data->tx_queues[i];
2973                         fm10k_txq_vec_setup(txq);
2974                 }
2975                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2976                 dev->tx_pkt_prepare = NULL;
2977         } else {
2978                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2979                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2980                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2981         }
2982 }
2983
2984 static void __rte_cold
2985 fm10k_set_rx_function(struct rte_eth_dev *dev)
2986 {
2987         struct fm10k_dev_info *dev_info =
2988                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2989         uint16_t i, rx_using_sse;
2990         uint16_t rx_ftag_en = 0;
2991
2992         if (fm10k_check_ftag(dev->device->devargs))
2993                 rx_ftag_en = 1;
2994
2995         /* In order to allow Vector Rx there are a few configuration
2996          * conditions to be met.
2997          */
2998         if (!fm10k_rx_vec_condition_check(dev) &&
2999                         dev_info->rx_vec_allowed && !rx_ftag_en &&
3000                         rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
3001                 if (dev->data->scattered_rx)
3002                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3003                 else
3004                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3005         } else if (dev->data->scattered_rx)
3006                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3007         else
3008                 dev->rx_pkt_burst = fm10k_recv_pkts;
3009
3010         rx_using_sse =
3011                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3012                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3013
3014         if (rx_using_sse)
3015                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3016         else
3017                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3018
3019         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3020                 return;
3021
3022         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3023                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3024
3025                 rxq->rx_using_sse = rx_using_sse;
3026                 rxq->rx_ftag_en = rx_ftag_en;
3027         }
3028 }
3029
3030 static void
3031 fm10k_params_init(struct rte_eth_dev *dev)
3032 {
3033         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3034         struct fm10k_dev_info *info =
3035                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3036
3037         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3038          * there is no way to get link status without reading BAR4.  Until this
3039          * works, assume we have maximum bandwidth.
3040          * @todo - fix bus info
3041          */
3042         hw->bus_caps.speed = fm10k_bus_speed_8000;
3043         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3044         hw->bus_caps.payload = fm10k_bus_payload_512;
3045         hw->bus.speed = fm10k_bus_speed_8000;
3046         hw->bus.width = fm10k_bus_width_pcie_x8;
3047         hw->bus.payload = fm10k_bus_payload_256;
3048
3049         info->rx_vec_allowed = true;
3050         info->sm_down = false;
3051 }
3052
3053 static int
3054 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3055 {
3056         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3057         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3058         struct rte_intr_handle *intr_handle = pdev->intr_handle;
3059         int diag, i;
3060         struct fm10k_macvlan_filter_info *macvlan;
3061
3062         PMD_INIT_FUNC_TRACE();
3063
3064         dev->dev_ops = &fm10k_eth_dev_ops;
3065         dev->rx_queue_count = fm10k_dev_rx_queue_count;
3066         dev->rx_descriptor_status = fm10k_dev_rx_descriptor_status;
3067         dev->tx_descriptor_status = fm10k_dev_tx_descriptor_status;
3068         dev->rx_pkt_burst = &fm10k_recv_pkts;
3069         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3070         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3071
3072         /*
3073          * Primary process does the whole initialization, for secondary
3074          * processes, we just select the same Rx and Tx function as primary.
3075          */
3076         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3077                 fm10k_set_rx_function(dev);
3078                 fm10k_set_tx_function(dev);
3079                 return 0;
3080         }
3081
3082         rte_eth_copy_pci_info(dev, pdev);
3083         dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
3084
3085         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3086         memset(macvlan, 0, sizeof(*macvlan));
3087         /* Vendor and Device ID need to be set before init of shared code */
3088         memset(hw, 0, sizeof(*hw));
3089         hw->device_id = pdev->id.device_id;
3090         hw->vendor_id = pdev->id.vendor_id;
3091         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3092         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3093         hw->revision_id = 0;
3094         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3095         if (hw->hw_addr == NULL) {
3096                 PMD_INIT_LOG(ERR, "Bad mem resource."
3097                         " Try to refuse unused devices.");
3098                 return -EIO;
3099         }
3100
3101         /* Store fm10k_adapter pointer */
3102         hw->back = dev->data->dev_private;
3103
3104         /* Initialize the shared code */
3105         diag = fm10k_init_shared_code(hw);
3106         if (diag != FM10K_SUCCESS) {
3107                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3108                 return -EIO;
3109         }
3110
3111         /* Initialize parameters */
3112         fm10k_params_init(dev);
3113
3114         /* Initialize the hw */
3115         diag = fm10k_init_hw(hw);
3116         if (diag != FM10K_SUCCESS) {
3117                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3118                 return -EIO;
3119         }
3120
3121         /* Initialize MAC address(es) */
3122         dev->data->mac_addrs = rte_zmalloc("fm10k",
3123                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3124         if (dev->data->mac_addrs == NULL) {
3125                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3126                 return -ENOMEM;
3127         }
3128
3129         diag = fm10k_read_mac_addr(hw);
3130
3131         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3132                         &dev->data->mac_addrs[0]);
3133
3134         if (diag != FM10K_SUCCESS ||
3135                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3136
3137                 /* Generate a random addr */
3138                 rte_eth_random_addr(hw->mac.addr);
3139                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3140                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3141                 &dev->data->mac_addrs[0]);
3142         }
3143
3144         /* Reset the hw statistics */
3145         diag = fm10k_stats_reset(dev);
3146         if (diag != 0) {
3147                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3148                 return diag;
3149         }
3150
3151         /* Reset the hw */
3152         diag = fm10k_reset_hw(hw);
3153         if (diag != FM10K_SUCCESS) {
3154                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3155                 return -EIO;
3156         }
3157
3158         /* Setup mailbox service */
3159         diag = fm10k_setup_mbx_service(hw);
3160         if (diag != FM10K_SUCCESS) {
3161                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3162                 return -EIO;
3163         }
3164
3165         /*PF/VF has different interrupt handling mechanism */
3166         if (hw->mac.type == fm10k_mac_pf) {
3167                 /* register callback func to eal lib */
3168                 rte_intr_callback_register(intr_handle,
3169                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3170
3171                 /* enable MISC interrupt */
3172                 fm10k_dev_enable_intr_pf(dev);
3173         } else { /* VF */
3174                 rte_intr_callback_register(intr_handle,
3175                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3176
3177                 fm10k_dev_enable_intr_vf(dev);
3178         }
3179
3180         /* Enable intr after callback registered */
3181         rte_intr_enable(intr_handle);
3182
3183         hw->mac.ops.update_int_moderator(hw);
3184
3185         /* Make sure Switch Manager is ready before going forward. */
3186         if (hw->mac.type == fm10k_mac_pf) {
3187                 bool switch_ready = false;
3188
3189                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3190                         fm10k_mbx_lock(hw);
3191                         hw->mac.ops.get_host_state(hw, &switch_ready);
3192                         fm10k_mbx_unlock(hw);
3193                         if (switch_ready == true)
3194                                 break;
3195                         /* Delay some time to acquire async LPORT_MAP info. */
3196                         rte_delay_us(WAIT_SWITCH_MSG_US);
3197                 }
3198
3199                 if (switch_ready == false) {
3200                         PMD_INIT_LOG(ERR, "switch is not ready");
3201                         return -1;
3202                 }
3203         }
3204
3205         /*
3206          * Below function will trigger operations on mailbox, acquire lock to
3207          * avoid race condition from interrupt handler. Operations on mailbox
3208          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3209          * will handle and generate an interrupt to our side. Then,  FIFO in
3210          * mailbox will be touched.
3211          */
3212         fm10k_mbx_lock(hw);
3213         /* Enable port first */
3214         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3215                                         MAX_LPORT_NUM, 1);
3216
3217         /* Set unicast mode by default. App can change to other mode in other
3218          * API func.
3219          */
3220         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3221                                         FM10K_XCAST_MODE_NONE);
3222
3223         fm10k_mbx_unlock(hw);
3224
3225         /* Make sure default VID is ready before going forward. */
3226         if (hw->mac.type == fm10k_mac_pf) {
3227                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3228                         if (hw->mac.default_vid)
3229                                 break;
3230                         /* Delay some time to acquire async port VLAN info. */
3231                         rte_delay_us(WAIT_SWITCH_MSG_US);
3232                 }
3233
3234                 if (!hw->mac.default_vid) {
3235                         PMD_INIT_LOG(ERR, "default VID is not ready");
3236                         return -1;
3237                 }
3238         }
3239
3240         /* Add default mac address */
3241         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3242                 MAIN_VSI_POOL_NUMBER);
3243
3244         return 0;
3245 }
3246
3247 static int
3248 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3249 {
3250         PMD_INIT_FUNC_TRACE();
3251         fm10k_dev_close(dev);
3252         return 0;
3253 }
3254
3255 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3256         struct rte_pci_device *pci_dev)
3257 {
3258         return rte_eth_dev_pci_generic_probe(pci_dev,
3259                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3260 }
3261
3262 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3263 {
3264         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3265 }
3266
3267 /*
3268  * The set of PCI devices this driver supports. This driver will enable both PF
3269  * and SRIOV-VF devices.
3270  */
3271 static const struct rte_pci_id pci_id_fm10k_map[] = {
3272         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3273         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3274         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3275         { .vendor_id = 0, /* sentinel */ },
3276 };
3277
3278 static struct rte_pci_driver rte_pmd_fm10k = {
3279         .id_table = pci_id_fm10k_map,
3280         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3281         .probe = eth_fm10k_pci_probe,
3282         .remove = eth_fm10k_pci_remove,
3283 };
3284
3285 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3286 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3287 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3288 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_init, init, NOTICE);
3289 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_driver, driver, NOTICE);
3290 #ifdef RTE_ETHDEV_DEBUG_RX
3291 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_rx, rx, DEBUG);
3292 #endif
3293 #ifdef RTE_ETHDEV_DEBUG_TX
3294 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_tx, tx, DEBUG);
3295 #endif