drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 __rte_weak int
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 __rte_weak uint16_t
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 __rte_weak uint16_t
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 __rte_weak int
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 __rte_weak void
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 __rte_weak void
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 __rte_weak int
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 __rte_weak uint16_t
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static int
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return 0;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS) {
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930                 return -EAGAIN;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int
 937 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 938 {
 939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         uint8_t mode;
 941         int status;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Return if it didn't acquire valid glort range */
 946         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 947                 return 0;
 948
 949         if (dev->data->all_multicast == 1)
 950                 mode = FM10K_XCAST_MODE_ALLMULTI;
 951         else
 952                 mode = FM10K_XCAST_MODE_NONE;
 953
 954         fm10k_mbx_lock(hw);
 955         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 956                                 mode);
 957         fm10k_mbx_unlock(hw);
 958
 959         if (status != FM10K_SUCCESS) {
 960                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 961                 return -EAGAIN;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static void
 968 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971         int status;
 972
 973         PMD_INIT_FUNC_TRACE();
 974
 975         /* Return if it didn't acquire valid glort range */
 976         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 977                 return;
 978
 979         /* If promiscuous mode is enabled, it doesn't make sense to enable
 980          * allmulticast and disable promiscuous since fm10k only can select
 981          * one of the modes.
 982          */
 983         if (dev->data->promiscuous) {
 984                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 985                         "needn't enable allmulticast");
 986                 return;
 987         }
 988
 989         fm10k_mbx_lock(hw);
 990         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 991                                 FM10K_XCAST_MODE_ALLMULTI);
 992         fm10k_mbx_unlock(hw);
 993
 994         if (status != FM10K_SUCCESS)
 995                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 996 }
 997
 998 static void
 999 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1000 {
1001         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1002         int status;
1003
1004         PMD_INIT_FUNC_TRACE();
1005
1006         /* Return if it didn't acquire valid glort range */
1007         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1008                 return;
1009
1010         if (dev->data->promiscuous) {
1011                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1012                         "since promisc mode is enabled");
1013                 return;
1014         }
1015
1016         fm10k_mbx_lock(hw);
1017         /* Change mode to unicast mode */
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_NONE);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1031         uint16_t nb_queue_pools;
1032         struct fm10k_macvlan_filter_info *macvlan;
1033
1034         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1035         nb_queue_pools = macvlan->nb_queue_pools;
1036         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1037         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1038
1039         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1040         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1041         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1042                         hw->mac.dglort_map;
1043         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1044         /* Configure VMDQ/RSS DGlort Decoder */
1045         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1046
1047         /* Flow Director configurations, only queue number is valid. */
1048         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1049         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1050                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1051         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1052         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1053
1054         /* Invalidate all other GLORT entries */
1055         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1056                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1057                                 FM10K_DGLORTMAP_NONE);
1058 }
1059
1060 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1061 static int
1062 fm10k_dev_start(struct rte_eth_dev *dev)
1063 {
1064         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1065         int i, diag;
1066
1067         PMD_INIT_FUNC_TRACE();
1068
1069         /* stop, init, then start the hw */
1070         diag = fm10k_stop_hw(hw);
1071         if (diag != FM10K_SUCCESS) {
1072                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1073                 return -EIO;
1074         }
1075
1076         diag = fm10k_init_hw(hw);
1077         if (diag != FM10K_SUCCESS) {
1078                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1079                 return -EIO;
1080         }
1081
1082         diag = fm10k_start_hw(hw);
1083         if (diag != FM10K_SUCCESS) {
1084                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1085                 return -EIO;
1086         }
1087
1088         diag = fm10k_dev_tx_init(dev);
1089         if (diag) {
1090                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1091                 return diag;
1092         }
1093
1094         if (fm10k_dev_rxq_interrupt_setup(dev))
1095                 return -EIO;
1096
1097         diag = fm10k_dev_rx_init(dev);
1098         if (diag) {
1099                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1100                 return diag;
1101         }
1102
1103         if (hw->mac.type == fm10k_mac_pf)
1104                 fm10k_dev_dglort_map_configure(dev);
1105
1106         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1107                 struct fm10k_rx_queue *rxq;
1108                 rxq = dev->data->rx_queues[i];
1109
1110                 if (rxq->rx_deferred_start)
1111                         continue;
1112                 diag = fm10k_dev_rx_queue_start(dev, i);
1113                 if (diag != 0) {
1114                         int j;
1115                         for (j = 0; j < i; ++j)
1116                                 rx_queue_clean(dev->data->rx_queues[j]);
1117                         return diag;
1118                 }
1119         }
1120
1121         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1122                 struct fm10k_tx_queue *txq;
1123                 txq = dev->data->tx_queues[i];
1124
1125                 if (txq->tx_deferred_start)
1126                         continue;
1127                 diag = fm10k_dev_tx_queue_start(dev, i);
1128                 if (diag != 0) {
1129                         int j;
1130                         for (j = 0; j < i; ++j)
1131                                 tx_queue_clean(dev->data->tx_queues[j]);
1132                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1133                                 rx_queue_clean(dev->data->rx_queues[j]);
1134                         return diag;
1135                 }
1136         }
1137
1138         /* Update default vlan when not in VMDQ mode */
1139         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1140                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1141
1142         fm10k_link_update(dev, 0);
1143
1144         return 0;
1145 }
1146
1147 static void
1148 fm10k_dev_stop(struct rte_eth_dev *dev)
1149 {
1150         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1151         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1152         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1153         int i;
1154
1155         PMD_INIT_FUNC_TRACE();
1156
1157         if (dev->data->tx_queues)
1158                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1159                         fm10k_dev_tx_queue_stop(dev, i);
1160
1161         if (dev->data->rx_queues)
1162                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1163                         fm10k_dev_rx_queue_stop(dev, i);
1164
1165         /* Disable datapath event */
1166         if (rte_intr_dp_is_en(intr_handle)) {
1167                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1168                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1169                                 3 << FM10K_RXINT_TIMER_SHIFT);
1170                         if (hw->mac.type == fm10k_mac_pf)
1171                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1172                                         FM10K_ITR_MASK_SET);
1173                         else
1174                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1175                                         FM10K_ITR_MASK_SET);
1176                 }
1177         }
1178         /* Clean datapath event and queue/vec mapping */
1179         rte_intr_efd_disable(intr_handle);
1180         rte_free(intr_handle->intr_vec);
1181         intr_handle->intr_vec = NULL;
1182 }
1183
1184 static void
1185 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1186 {
1187         int i;
1188
1189         PMD_INIT_FUNC_TRACE();
1190
1191         if (dev->data->tx_queues) {
1192                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1193                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1194
1195                         tx_queue_free(txq);
1196                 }
1197         }
1198
1199         if (dev->data->rx_queues) {
1200                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1201                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1202         }
1203 }
1204
1205 static void
1206 fm10k_dev_close(struct rte_eth_dev *dev)
1207 {
1208         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1209
1210         PMD_INIT_FUNC_TRACE();
1211
1212         fm10k_mbx_lock(hw);
1213         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1214                 MAX_LPORT_NUM, false);
1215         fm10k_mbx_unlock(hw);
1216
1217         /* allow 100ms for device to quiesce */
1218         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1219
1220         /* Stop mailbox service first */
1221         fm10k_close_mbx_service(hw);
1222         fm10k_dev_stop(dev);
1223         fm10k_dev_queue_release(dev);
1224         fm10k_stop_hw(hw);
1225 }
1226
1227 static int
1228 fm10k_link_update(struct rte_eth_dev *dev,
1229         __rte_unused int wait_to_complete)
1230 {
1231         struct fm10k_dev_info *dev_info =
1232                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1233         PMD_INIT_FUNC_TRACE();
1234
1235         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1236         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1237         dev->data->dev_link.link_status =
1238                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1239         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1240
1241         return 0;
1242 }
1243
1244 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1245         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1246 {
1247         unsigned i, q;
1248         unsigned count = 0;
1249
1250         if (xstats_names != NULL) {
1251                 /* Note: limit checked in rte_eth_xstats_names() */
1252
1253                 /* Global stats */
1254                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1255                         snprintf(xstats_names[count].name,
1256                                 sizeof(xstats_names[count].name),
1257                                 "%s", fm10k_hw_stats_strings[count].name);
1258                         count++;
1259                 }
1260
1261                 /* PF queue stats */
1262                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1263                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1264                                 snprintf(xstats_names[count].name,
1265                                         sizeof(xstats_names[count].name),
1266                                         "rx_q%u_%s", q,
1267                                         fm10k_hw_stats_rx_q_strings[i].name);
1268                                 count++;
1269                         }
1270                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1271                                 snprintf(xstats_names[count].name,
1272                                         sizeof(xstats_names[count].name),
1273                                         "tx_q%u_%s", q,
1274                                         fm10k_hw_stats_tx_q_strings[i].name);
1275                                 count++;
1276                         }
1277                 }
1278         }
1279         return FM10K_NB_XSTATS;
1280 }
1281
1282 static int
1283 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1284                  unsigned n)
1285 {
1286         struct fm10k_hw_stats *hw_stats =
1287                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1288         unsigned i, q, count = 0;
1289
1290         if (n < FM10K_NB_XSTATS)
1291                 return FM10K_NB_XSTATS;
1292
1293         /* Global stats */
1294         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1295                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1296                         fm10k_hw_stats_strings[count].offset);
1297                 xstats[count].id = count;
1298                 count++;
1299         }
1300
1301         /* PF queue stats */
1302         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1303                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1304                         xstats[count].value =
1305                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1306                                 fm10k_hw_stats_rx_q_strings[i].offset);
1307                         xstats[count].id = count;
1308                         count++;
1309                 }
1310                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1311                         xstats[count].value =
1312                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1313                                 fm10k_hw_stats_tx_q_strings[i].offset);
1314                         xstats[count].id = count;
1315                         count++;
1316                 }
1317         }
1318
1319         return FM10K_NB_XSTATS;
1320 }
1321
1322 static int
1323 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1324 {
1325         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1326         struct fm10k_hw *hw =
1327                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1328         struct fm10k_hw_stats *hw_stats =
1329                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1330         int i;
1331
1332         PMD_INIT_FUNC_TRACE();
1333
1334         fm10k_update_hw_stats(hw, hw_stats);
1335
1336         ipackets = opackets = ibytes = obytes = imissed = 0;
1337         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1338                 (i < hw->mac.max_queues); ++i) {
1339                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1340                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1341                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1342                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1343                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1344                 ipackets += stats->q_ipackets[i];
1345                 opackets += stats->q_opackets[i];
1346                 ibytes   += stats->q_ibytes[i];
1347                 obytes   += stats->q_obytes[i];
1348                 imissed  += stats->q_errors[i];
1349         }
1350         stats->ipackets = ipackets;
1351         stats->opackets = opackets;
1352         stats->ibytes = ibytes;
1353         stats->obytes = obytes;
1354         stats->imissed = imissed;
1355         return 0;
1356 }
1357
1358 static int
1359 fm10k_stats_reset(struct rte_eth_dev *dev)
1360 {
1361         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1362         struct fm10k_hw_stats *hw_stats =
1363                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1364
1365         PMD_INIT_FUNC_TRACE();
1366
1367         memset(hw_stats, 0, sizeof(*hw_stats));
1368         fm10k_rebind_hw_stats(hw, hw_stats);
1369
1370         return 0;
1371 }
1372
1373 static int
1374 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1375         struct rte_eth_dev_info *dev_info)
1376 {
1377         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1378         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1379
1380         PMD_INIT_FUNC_TRACE();
1381
1382         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1383         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1384         dev_info->max_rx_queues      = hw->mac.max_queues;
1385         dev_info->max_tx_queues      = hw->mac.max_queues;
1386         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1387         dev_info->max_hash_mac_addrs = 0;
1388         dev_info->max_vfs            = pdev->max_vfs;
1389         dev_info->vmdq_pool_base     = 0;
1390         dev_info->vmdq_queue_base    = 0;
1391         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1392         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1393         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1394         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1395                                     dev_info->rx_queue_offload_capa;
1396         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1397         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1398                                     dev_info->tx_queue_offload_capa;
1399
1400         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1401         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1402         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1403                                         ETH_RSS_IPV6 |
1404                                         ETH_RSS_IPV6_EX |
1405                                         ETH_RSS_NONFRAG_IPV4_TCP |
1406                                         ETH_RSS_NONFRAG_IPV6_TCP |
1407                                         ETH_RSS_IPV6_TCP_EX |
1408                                         ETH_RSS_NONFRAG_IPV4_UDP |
1409                                         ETH_RSS_NONFRAG_IPV6_UDP |
1410                                         ETH_RSS_IPV6_UDP_EX;
1411
1412         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1413                 .rx_thresh = {
1414                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1415                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1416                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1417                 },
1418                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1419                 .rx_drop_en = 0,
1420                 .offloads = 0,
1421         };
1422
1423         dev_info->default_txconf = (struct rte_eth_txconf) {
1424                 .tx_thresh = {
1425                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1426                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1427                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1428                 },
1429                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1430                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1431                 .offloads = 0,
1432         };
1433
1434         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1435                 .nb_max = FM10K_MAX_RX_DESC,
1436                 .nb_min = FM10K_MIN_RX_DESC,
1437                 .nb_align = FM10K_MULT_RX_DESC,
1438         };
1439
1440         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1441                 .nb_max = FM10K_MAX_TX_DESC,
1442                 .nb_min = FM10K_MIN_TX_DESC,
1443                 .nb_align = FM10K_MULT_TX_DESC,
1444                 .nb_seg_max = FM10K_TX_MAX_SEG,
1445                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1446         };
1447
1448         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1449                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1450                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1451
1452         return 0;
1453 }
1454
1455 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1456 static const uint32_t *
1457 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1458 {
1459         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1460             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1461                 static uint32_t ptypes[] = {
1462                         /* refers to rx_desc_to_ol_flags() */
1463                         RTE_PTYPE_L2_ETHER,
1464                         RTE_PTYPE_L3_IPV4,
1465                         RTE_PTYPE_L3_IPV4_EXT,
1466                         RTE_PTYPE_L3_IPV6,
1467                         RTE_PTYPE_L3_IPV6_EXT,
1468                         RTE_PTYPE_L4_TCP,
1469                         RTE_PTYPE_L4_UDP,
1470                         RTE_PTYPE_UNKNOWN
1471                 };
1472
1473                 return ptypes;
1474         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1475                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1476                 static uint32_t ptypes_vec[] = {
1477                         /* refers to fm10k_desc_to_pktype_v() */
1478                         RTE_PTYPE_L3_IPV4,
1479                         RTE_PTYPE_L3_IPV4_EXT,
1480                         RTE_PTYPE_L3_IPV6,
1481                         RTE_PTYPE_L3_IPV6_EXT,
1482                         RTE_PTYPE_L4_TCP,
1483                         RTE_PTYPE_L4_UDP,
1484                         RTE_PTYPE_TUNNEL_GENEVE,
1485                         RTE_PTYPE_TUNNEL_NVGRE,
1486                         RTE_PTYPE_TUNNEL_VXLAN,
1487                         RTE_PTYPE_TUNNEL_GRE,
1488                         RTE_PTYPE_UNKNOWN
1489                 };
1490
1491                 return ptypes_vec;
1492         }
1493
1494         return NULL;
1495 }
1496 #else
1497 static const uint32_t *
1498 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1499 {
1500         return NULL;
1501 }
1502 #endif
1503
1504 static int
1505 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1506 {
1507         s32 result;
1508         uint16_t mac_num = 0;
1509         uint32_t vid_idx, vid_bit, mac_index;
1510         struct fm10k_hw *hw;
1511         struct fm10k_macvlan_filter_info *macvlan;
1512         struct rte_eth_dev_data *data = dev->data;
1513
1514         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1515         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1516
1517         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1518                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1519                 return -EINVAL;
1520         }
1521
1522         if (vlan_id > ETH_VLAN_ID_MAX) {
1523                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1524                 return -EINVAL;
1525         }
1526
1527         vid_idx = FM10K_VFTA_IDX(vlan_id);
1528         vid_bit = FM10K_VFTA_BIT(vlan_id);
1529         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1530         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1531                 return 0;
1532         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1533         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1534                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1535                         "in the VLAN filter table");
1536                 return -EINVAL;
1537         }
1538
1539         fm10k_mbx_lock(hw);
1540         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1541         fm10k_mbx_unlock(hw);
1542         if (result != FM10K_SUCCESS) {
1543                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1544                 return -EIO;
1545         }
1546
1547         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1548                         (result == FM10K_SUCCESS); mac_index++) {
1549                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1550                         continue;
1551                 if (mac_num > macvlan->mac_num - 1) {
1552                         PMD_INIT_LOG(ERR, "MAC address number "
1553                                         "not match");
1554                         break;
1555                 }
1556                 fm10k_mbx_lock(hw);
1557                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1558                         data->mac_addrs[mac_index].addr_bytes,
1559                         vlan_id, on, 0);
1560                 fm10k_mbx_unlock(hw);
1561                 mac_num++;
1562         }
1563         if (result != FM10K_SUCCESS) {
1564                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1565                 return -EIO;
1566         }
1567
1568         if (on) {
1569                 macvlan->vlan_num++;
1570                 macvlan->vfta[vid_idx] |= vid_bit;
1571         } else {
1572                 macvlan->vlan_num--;
1573                 macvlan->vfta[vid_idx] &= ~vid_bit;
1574         }
1575         return 0;
1576 }
1577
1578 static int
1579 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1580 {
1581         if (mask & ETH_VLAN_STRIP_MASK) {
1582                 if (!(dev->data->dev_conf.rxmode.offloads &
1583                         DEV_RX_OFFLOAD_VLAN_STRIP))
1584                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1585                                         "always on in fm10k");
1586         }
1587
1588         if (mask & ETH_VLAN_EXTEND_MASK) {
1589                 if (dev->data->dev_conf.rxmode.offloads &
1590                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1591                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1592                                         "supported in fm10k");
1593         }
1594
1595         if (mask & ETH_VLAN_FILTER_MASK) {
1596                 if (!(dev->data->dev_conf.rxmode.offloads &
1597                         DEV_RX_OFFLOAD_VLAN_FILTER))
1598                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1599         }
1600
1601         return 0;
1602 }
1603
1604 /* Add/Remove a MAC address, and update filters to main VSI */
1605 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1606                 const u8 *mac, bool add, uint32_t pool)
1607 {
1608         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1609         struct fm10k_macvlan_filter_info *macvlan;
1610         uint32_t i, j, k;
1611
1612         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1613
1614         if (pool != MAIN_VSI_POOL_NUMBER) {
1615                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1616                         "mac to pool %u", pool);
1617                 return;
1618         }
1619         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1620                 if (!macvlan->vfta[j])
1621                         continue;
1622                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1623                         if (!(macvlan->vfta[j] & (1 << k)))
1624                                 continue;
1625                         if (i + 1 > macvlan->vlan_num) {
1626                                 PMD_INIT_LOG(ERR, "vlan number not match");
1627                                 return;
1628                         }
1629                         fm10k_mbx_lock(hw);
1630                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1631                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1632                         fm10k_mbx_unlock(hw);
1633                         i++;
1634                 }
1635         }
1636 }
1637
1638 /* Add/Remove a MAC address, and update filters to VMDQ */
1639 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1640                 const u8 *mac, bool add, uint32_t pool)
1641 {
1642         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1643         struct fm10k_macvlan_filter_info *macvlan;
1644         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1645         uint32_t i;
1646
1647         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1648         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1649
1650         if (pool > macvlan->nb_queue_pools) {
1651                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1652                         " Max pool is %u",
1653                         pool, macvlan->nb_queue_pools);
1654                 return;
1655         }
1656         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1657                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1658                         continue;
1659                 fm10k_mbx_lock(hw);
1660                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1661                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1662                 fm10k_mbx_unlock(hw);
1663         }
1664 }
1665
1666 /* Add/Remove a MAC address, and update filters */
1667 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1668                 const u8 *mac, bool add, uint32_t pool)
1669 {
1670         struct fm10k_macvlan_filter_info *macvlan;
1671
1672         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1673
1674         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1675                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1676         else
1677                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1678
1679         if (add)
1680                 macvlan->mac_num++;
1681         else
1682                 macvlan->mac_num--;
1683 }
1684
1685 /* Add a MAC address, and update filters */
1686 static int
1687 fm10k_macaddr_add(struct rte_eth_dev *dev,
1688                 struct rte_ether_addr *mac_addr,
1689                 uint32_t index,
1690                 uint32_t pool)
1691 {
1692         struct fm10k_macvlan_filter_info *macvlan;
1693
1694         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1695         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1696         macvlan->mac_vmdq_id[index] = pool;
1697         return 0;
1698 }
1699
1700 /* Remove a MAC address, and update filters */
1701 static void
1702 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1703 {
1704         struct rte_eth_dev_data *data = dev->data;
1705         struct fm10k_macvlan_filter_info *macvlan;
1706
1707         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1708         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1709                         FALSE, macvlan->mac_vmdq_id[index]);
1710         macvlan->mac_vmdq_id[index] = 0;
1711 }
1712
1713 static inline int
1714 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1715 {
1716         if ((request < min) || (request > max) || ((request % mult) != 0))
1717                 return -1;
1718         else
1719                 return 0;
1720 }
1721
1722
1723 static inline int
1724 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1725 {
1726         if ((request < min) || (request > max) || ((div % request) != 0))
1727                 return -1;
1728         else
1729                 return 0;
1730 }
1731
1732 static inline int
1733 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1734 {
1735         uint16_t rx_free_thresh;
1736
1737         if (conf->rx_free_thresh == 0)
1738                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1739         else
1740                 rx_free_thresh = conf->rx_free_thresh;
1741
1742         /* make sure the requested threshold satisfies the constraints */
1743         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1744                         FM10K_RX_FREE_THRESH_MAX(q),
1745                         FM10K_RX_FREE_THRESH_DIV(q),
1746                         rx_free_thresh)) {
1747                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1748                         "less than or equal to %u, "
1749                         "greater than or equal to %u, "
1750                         "and a divisor of %u",
1751                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1752                         FM10K_RX_FREE_THRESH_MIN(q),
1753                         FM10K_RX_FREE_THRESH_DIV(q));
1754                 return -EINVAL;
1755         }
1756
1757         q->alloc_thresh = rx_free_thresh;
1758         q->drop_en = conf->rx_drop_en;
1759         q->rx_deferred_start = conf->rx_deferred_start;
1760
1761         return 0;
1762 }
1763
1764 /*
1765  * Hardware requires specific alignment for Rx packet buffers. At
1766  * least one of the following two conditions must be satisfied.
1767  *  1. Address is 512B aligned
1768  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1769  *
1770  * As such, the driver may need to adjust the DMA address within the
1771  * buffer by up to 512B.
1772  *
1773  * return 1 if the element size is valid, otherwise return 0.
1774  */
1775 static int
1776 mempool_element_size_valid(struct rte_mempool *mp)
1777 {
1778         uint32_t min_size;
1779
1780         /* elt_size includes mbuf header and headroom */
1781         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1782                         RTE_PKTMBUF_HEADROOM;
1783
1784         /* account for up to 512B of alignment */
1785         min_size -= FM10K_RX_DATABUF_ALIGN;
1786
1787         /* sanity check for overflow */
1788         if (min_size > mp->elt_size)
1789                 return 0;
1790
1791         /* size is valid */
1792         return 1;
1793 }
1794
1795 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1796 {
1797         RTE_SET_USED(dev);
1798
1799         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1800 }
1801
1802 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1803 {
1804         RTE_SET_USED(dev);
1805
1806         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1807                            DEV_RX_OFFLOAD_VLAN_FILTER |
1808                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1809                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1810                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1811                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1812                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1813 }
1814
1815 static int
1816 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1817         uint16_t nb_desc, unsigned int socket_id,
1818         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1819 {
1820         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1821         struct fm10k_dev_info *dev_info =
1822                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1823         struct fm10k_rx_queue *q;
1824         const struct rte_memzone *mz;
1825         uint64_t offloads;
1826
1827         PMD_INIT_FUNC_TRACE();
1828
1829         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1830
1831         /* make sure the mempool element size can account for alignment. */
1832         if (!mempool_element_size_valid(mp)) {
1833                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1834                 return -EINVAL;
1835         }
1836
1837         /* make sure a valid number of descriptors have been requested */
1838         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1839                                 FM10K_MULT_RX_DESC, nb_desc)) {
1840                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1841                         "less than or equal to %"PRIu32", "
1842                         "greater than or equal to %u, "
1843                         "and a multiple of %u",
1844                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1845                         FM10K_MULT_RX_DESC);
1846                 return -EINVAL;
1847         }
1848
1849         /*
1850          * if this queue existed already, free the associated memory. The
1851          * queue cannot be reused in case we need to allocate memory on
1852          * different socket than was previously used.
1853          */
1854         if (dev->data->rx_queues[queue_id] != NULL) {
1855                 rx_queue_free(dev->data->rx_queues[queue_id]);
1856                 dev->data->rx_queues[queue_id] = NULL;
1857         }
1858
1859         /* allocate memory for the queue structure */
1860         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1861                                 socket_id);
1862         if (q == NULL) {
1863                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1864                 return -ENOMEM;
1865         }
1866
1867         /* setup queue */
1868         q->mp = mp;
1869         q->nb_desc = nb_desc;
1870         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1871         q->port_id = dev->data->port_id;
1872         q->queue_id = queue_id;
1873         q->tail_ptr = (volatile uint32_t *)
1874                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1875         q->offloads = offloads;
1876         if (handle_rxconf(q, conf))
1877                 return -EINVAL;
1878
1879         /* allocate memory for the software ring */
1880         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1881                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1882                         RTE_CACHE_LINE_SIZE, socket_id);
1883         if (q->sw_ring == NULL) {
1884                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1885                 rte_free(q);
1886                 return -ENOMEM;
1887         }
1888
1889         /*
1890          * allocate memory for the hardware descriptor ring. A memzone large
1891          * enough to hold the maximum ring size is requested to allow for
1892          * resizing in later calls to the queue setup function.
1893          */
1894         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1895                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1896                                       socket_id);
1897         if (mz == NULL) {
1898                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1899                 rte_free(q->sw_ring);
1900                 rte_free(q);
1901                 return -ENOMEM;
1902         }
1903         q->hw_ring = mz->addr;
1904         q->hw_ring_phys_addr = mz->iova;
1905
1906         /* Check if number of descs satisfied Vector requirement */
1907         if (!rte_is_power_of_2(nb_desc)) {
1908                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1909                                     "preconditions - canceling the feature for "
1910                                     "the whole port[%d]",
1911                              q->queue_id, q->port_id);
1912                 dev_info->rx_vec_allowed = false;
1913         } else
1914                 fm10k_rxq_vec_setup(q);
1915
1916         dev->data->rx_queues[queue_id] = q;
1917         return 0;
1918 }
1919
1920 static void
1921 fm10k_rx_queue_release(void *queue)
1922 {
1923         PMD_INIT_FUNC_TRACE();
1924
1925         rx_queue_free(queue);
1926 }
1927
1928 static inline int
1929 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1930 {
1931         uint16_t tx_free_thresh;
1932         uint16_t tx_rs_thresh;
1933
1934         /* constraint MACROs require that tx_free_thresh is configured
1935          * before tx_rs_thresh */
1936         if (conf->tx_free_thresh == 0)
1937                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1938         else
1939                 tx_free_thresh = conf->tx_free_thresh;
1940
1941         /* make sure the requested threshold satisfies the constraints */
1942         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1943                         FM10K_TX_FREE_THRESH_MAX(q),
1944                         FM10K_TX_FREE_THRESH_DIV(q),
1945                         tx_free_thresh)) {
1946                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1947                         "less than or equal to %u, "
1948                         "greater than or equal to %u, "
1949                         "and a divisor of %u",
1950                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1951                         FM10K_TX_FREE_THRESH_MIN(q),
1952                         FM10K_TX_FREE_THRESH_DIV(q));
1953                 return -EINVAL;
1954         }
1955
1956         q->free_thresh = tx_free_thresh;
1957
1958         if (conf->tx_rs_thresh == 0)
1959                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1960         else
1961                 tx_rs_thresh = conf->tx_rs_thresh;
1962
1963         q->tx_deferred_start = conf->tx_deferred_start;
1964
1965         /* make sure the requested threshold satisfies the constraints */
1966         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1967                         FM10K_TX_RS_THRESH_MAX(q),
1968                         FM10K_TX_RS_THRESH_DIV(q),
1969                         tx_rs_thresh)) {
1970                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1971                         "less than or equal to %u, "
1972                         "greater than or equal to %u, "
1973                         "and a divisor of %u",
1974                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1975                         FM10K_TX_RS_THRESH_MIN(q),
1976                         FM10K_TX_RS_THRESH_DIV(q));
1977                 return -EINVAL;
1978         }
1979
1980         q->rs_thresh = tx_rs_thresh;
1981
1982         return 0;
1983 }
1984
1985 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1986 {
1987         RTE_SET_USED(dev);
1988
1989         return 0;
1990 }
1991
1992 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1993 {
1994         RTE_SET_USED(dev);
1995
1996         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1997                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1998                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1999                           DEV_TX_OFFLOAD_UDP_CKSUM   |
2000                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2001                           DEV_TX_OFFLOAD_TCP_TSO);
2002 }
2003
2004 static int
2005 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2006         uint16_t nb_desc, unsigned int socket_id,
2007         const struct rte_eth_txconf *conf)
2008 {
2009         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2010         struct fm10k_tx_queue *q;
2011         const struct rte_memzone *mz;
2012         uint64_t offloads;
2013
2014         PMD_INIT_FUNC_TRACE();
2015
2016         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2017
2018         /* make sure a valid number of descriptors have been requested */
2019         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2020                                 FM10K_MULT_TX_DESC, nb_desc)) {
2021                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2022                         "less than or equal to %"PRIu32", "
2023                         "greater than or equal to %u, "
2024                         "and a multiple of %u",
2025                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2026                         FM10K_MULT_TX_DESC);
2027                 return -EINVAL;
2028         }
2029
2030         /*
2031          * if this queue existed already, free the associated memory. The
2032          * queue cannot be reused in case we need to allocate memory on
2033          * different socket than was previously used.
2034          */
2035         if (dev->data->tx_queues[queue_id] != NULL) {
2036                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2037
2038                 tx_queue_free(txq);
2039                 dev->data->tx_queues[queue_id] = NULL;
2040         }
2041
2042         /* allocate memory for the queue structure */
2043         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2044                                 socket_id);
2045         if (q == NULL) {
2046                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2047                 return -ENOMEM;
2048         }
2049
2050         /* setup queue */
2051         q->nb_desc = nb_desc;
2052         q->port_id = dev->data->port_id;
2053         q->queue_id = queue_id;
2054         q->offloads = offloads;
2055         q->ops = &def_txq_ops;
2056         q->tail_ptr = (volatile uint32_t *)
2057                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2058         if (handle_txconf(q, conf))
2059                 return -EINVAL;
2060
2061         /* allocate memory for the software ring */
2062         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2063                                         nb_desc * sizeof(struct rte_mbuf *),
2064                                         RTE_CACHE_LINE_SIZE, socket_id);
2065         if (q->sw_ring == NULL) {
2066                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2067                 rte_free(q);
2068                 return -ENOMEM;
2069         }
2070
2071         /*
2072          * allocate memory for the hardware descriptor ring. A memzone large
2073          * enough to hold the maximum ring size is requested to allow for
2074          * resizing in later calls to the queue setup function.
2075          */
2076         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2077                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2078                                       socket_id);
2079         if (mz == NULL) {
2080                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2081                 rte_free(q->sw_ring);
2082                 rte_free(q);
2083                 return -ENOMEM;
2084         }
2085         q->hw_ring = mz->addr;
2086         q->hw_ring_phys_addr = mz->iova;
2087
2088         /*
2089          * allocate memory for the RS bit tracker. Enough slots to hold the
2090          * descriptor index for each RS bit needing to be set are required.
2091          */
2092         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2093                                 ((nb_desc + 1) / q->rs_thresh) *
2094                                 sizeof(uint16_t),
2095                                 RTE_CACHE_LINE_SIZE, socket_id);
2096         if (q->rs_tracker.list == NULL) {
2097                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2098                 rte_free(q->sw_ring);
2099                 rte_free(q);
2100                 return -ENOMEM;
2101         }
2102
2103         dev->data->tx_queues[queue_id] = q;
2104         return 0;
2105 }
2106
2107 static void
2108 fm10k_tx_queue_release(void *queue)
2109 {
2110         struct fm10k_tx_queue *q = queue;
2111         PMD_INIT_FUNC_TRACE();
2112
2113         tx_queue_free(q);
2114 }
2115
2116 static int
2117 fm10k_reta_update(struct rte_eth_dev *dev,
2118                         struct rte_eth_rss_reta_entry64 *reta_conf,
2119                         uint16_t reta_size)
2120 {
2121         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2122         uint16_t i, j, idx, shift;
2123         uint8_t mask;
2124         uint32_t reta;
2125
2126         PMD_INIT_FUNC_TRACE();
2127
2128         if (reta_size > FM10K_MAX_RSS_INDICES) {
2129                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2130                         "(%d) doesn't match the number hardware can supported "
2131                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2132                 return -EINVAL;
2133         }
2134
2135         /*
2136          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2137          * 128-entries in 32 registers
2138          */
2139         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2140                 idx = i / RTE_RETA_GROUP_SIZE;
2141                 shift = i % RTE_RETA_GROUP_SIZE;
2142                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2143                                 BIT_MASK_PER_UINT32);
2144                 if (mask == 0)
2145                         continue;
2146
2147                 reta = 0;
2148                 if (mask != BIT_MASK_PER_UINT32)
2149                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2150
2151                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2152                         if (mask & (0x1 << j)) {
2153                                 if (mask != 0xF)
2154                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2155                                 reta |= reta_conf[idx].reta[shift + j] <<
2156                                                 (CHAR_BIT * j);
2157                         }
2158                 }
2159                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2160         }
2161
2162         return 0;
2163 }
2164
2165 static int
2166 fm10k_reta_query(struct rte_eth_dev *dev,
2167                         struct rte_eth_rss_reta_entry64 *reta_conf,
2168                         uint16_t reta_size)
2169 {
2170         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2171         uint16_t i, j, idx, shift;
2172         uint8_t mask;
2173         uint32_t reta;
2174
2175         PMD_INIT_FUNC_TRACE();
2176
2177         if (reta_size < FM10K_MAX_RSS_INDICES) {
2178                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2179                         "(%d) doesn't match the number hardware can supported "
2180                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2181                 return -EINVAL;
2182         }
2183
2184         /*
2185          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2186          * 128-entries in 32 registers
2187          */
2188         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2189                 idx = i / RTE_RETA_GROUP_SIZE;
2190                 shift = i % RTE_RETA_GROUP_SIZE;
2191                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2192                                 BIT_MASK_PER_UINT32);
2193                 if (mask == 0)
2194                         continue;
2195
2196                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2197                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2198                         if (mask & (0x1 << j))
2199                                 reta_conf[idx].reta[shift + j] = ((reta >>
2200                                         CHAR_BIT * j) & UINT8_MAX);
2201                 }
2202         }
2203
2204         return 0;
2205 }
2206
2207 static int
2208 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2209         struct rte_eth_rss_conf *rss_conf)
2210 {
2211         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2212         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2213         uint32_t mrqc;
2214         uint64_t hf = rss_conf->rss_hf;
2215         int i;
2216
2217         PMD_INIT_FUNC_TRACE();
2218
2219         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2220                                 FM10K_RSSRK_ENTRIES_PER_REG))
2221                 return -EINVAL;
2222
2223         if (hf == 0)
2224                 return -EINVAL;
2225
2226         mrqc = 0;
2227         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2228         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2229         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2230         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2231         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2232         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2233         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2234         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2235         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2236
2237         /* If the mapping doesn't fit any supported, return */
2238         if (mrqc == 0)
2239                 return -EINVAL;
2240
2241         if (key != NULL)
2242                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2243                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2244
2245         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2246
2247         return 0;
2248 }
2249
2250 static int
2251 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2252         struct rte_eth_rss_conf *rss_conf)
2253 {
2254         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2255         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2256         uint32_t mrqc;
2257         uint64_t hf;
2258         int i;
2259
2260         PMD_INIT_FUNC_TRACE();
2261
2262         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2263                                 FM10K_RSSRK_ENTRIES_PER_REG))
2264                 return -EINVAL;
2265
2266         if (key != NULL)
2267                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2268                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2269
2270         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2271         hf = 0;
2272         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2273         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2274         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2275         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2276         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2277         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2278         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2279         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2280         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2281
2282         rss_conf->rss_hf = hf;
2283
2284         return 0;
2285 }
2286
2287 static void
2288 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2289 {
2290         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2291         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2292
2293         /* Bind all local non-queue interrupt to vector 0 */
2294         int_map |= FM10K_MISC_VEC_ID;
2295
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2298         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2299         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2300         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2301         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2302
2303         /* Enable misc causes */
2304         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2305                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2306                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2307                                 FM10K_EIMR_ENABLE(MAILBOX) |
2308                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2309                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2310                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2311                                 FM10K_EIMR_ENABLE(VFLR));
2312
2313         /* Enable ITR 0 */
2314         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2315                                         FM10K_ITR_MASK_CLEAR);
2316         FM10K_WRITE_FLUSH(hw);
2317 }
2318
2319 static void
2320 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2321 {
2322         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2323         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2324
2325         int_map |= FM10K_MISC_VEC_ID;
2326
2327         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2328         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2329         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2330         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2331         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2332         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2333
2334         /* Disable misc causes */
2335         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2336                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2337                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2338                                 FM10K_EIMR_DISABLE(MAILBOX) |
2339                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2340                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2341                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2342                                 FM10K_EIMR_DISABLE(VFLR));
2343
2344         /* Disable ITR 0 */
2345         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2346         FM10K_WRITE_FLUSH(hw);
2347 }
2348
2349 static void
2350 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2351 {
2352         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2353         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2354
2355         /* Bind all local non-queue interrupt to vector 0 */
2356         int_map |= FM10K_MISC_VEC_ID;
2357
2358         /* Only INT 0 available, other 15 are reserved. */
2359         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2360
2361         /* Enable ITR 0 */
2362         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2363                                         FM10K_ITR_MASK_CLEAR);
2364         FM10K_WRITE_FLUSH(hw);
2365 }
2366
2367 static void
2368 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2369 {
2370         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2371         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2372
2373         int_map |= FM10K_MISC_VEC_ID;
2374
2375         /* Only INT 0 available, other 15 are reserved. */
2376         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2377
2378         /* Disable ITR 0 */
2379         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2380         FM10K_WRITE_FLUSH(hw);
2381 }
2382
2383 static int
2384 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2385 {
2386         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2387         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2388
2389         /* Enable ITR */
2390         if (hw->mac.type == fm10k_mac_pf)
2391                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2392                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2393         else
2394                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2395                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2396         rte_intr_ack(&pdev->intr_handle);
2397         return 0;
2398 }
2399
2400 static int
2401 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2402 {
2403         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2404         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2405
2406         /* Disable ITR */
2407         if (hw->mac.type == fm10k_mac_pf)
2408                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2409                         FM10K_ITR_MASK_SET);
2410         else
2411                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2412                         FM10K_ITR_MASK_SET);
2413         return 0;
2414 }
2415
2416 static int
2417 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2418 {
2419         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2420         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2421         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2422         uint32_t intr_vector, vec;
2423         uint16_t queue_id;
2424         int result = 0;
2425
2426         /* fm10k needs one separate interrupt for mailbox,
2427          * so only drivers which support multiple interrupt vectors
2428          * e.g. vfio-pci can work for fm10k interrupt mode
2429          */
2430         if (!rte_intr_cap_multiple(intr_handle) ||
2431                         dev->data->dev_conf.intr_conf.rxq == 0)
2432                 return result;
2433
2434         intr_vector = dev->data->nb_rx_queues;
2435
2436         /* disable interrupt first */
2437         rte_intr_disable(intr_handle);
2438         if (hw->mac.type == fm10k_mac_pf)
2439                 fm10k_dev_disable_intr_pf(dev);
2440         else
2441                 fm10k_dev_disable_intr_vf(dev);
2442
2443         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2444                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2445                 result = -EIO;
2446         }
2447
2448         if (rte_intr_dp_is_en(intr_handle) && !result) {
2449                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2450                         dev->data->nb_rx_queues * sizeof(int), 0);
2451                 if (intr_handle->intr_vec) {
2452                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2453                                         queue_id < dev->data->nb_rx_queues;
2454                                         queue_id++) {
2455                                 intr_handle->intr_vec[queue_id] = vec;
2456                                 if (vec < intr_handle->nb_efd - 1
2457                                                 + FM10K_RX_VEC_START)
2458                                         vec++;
2459                         }
2460                 } else {
2461                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2462                                 " intr_vec", dev->data->nb_rx_queues);
2463                         rte_intr_efd_disable(intr_handle);
2464                         result = -ENOMEM;
2465                 }
2466         }
2467
2468         if (hw->mac.type == fm10k_mac_pf)
2469                 fm10k_dev_enable_intr_pf(dev);
2470         else
2471                 fm10k_dev_enable_intr_vf(dev);
2472         rte_intr_enable(intr_handle);
2473         hw->mac.ops.update_int_moderator(hw);
2474         return result;
2475 }
2476
2477 static int
2478 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2479 {
2480         struct fm10k_fault fault;
2481         int err;
2482         const char *estr = "Unknown error";
2483
2484         /* Process PCA fault */
2485         if (eicr & FM10K_EICR_PCA_FAULT) {
2486                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2487                 if (err)
2488                         goto error;
2489                 switch (fault.type) {
2490                 case PCA_NO_FAULT:
2491                         estr = "PCA_NO_FAULT"; break;
2492                 case PCA_UNMAPPED_ADDR:
2493                         estr = "PCA_UNMAPPED_ADDR"; break;
2494                 case PCA_BAD_QACCESS_PF:
2495                         estr = "PCA_BAD_QACCESS_PF"; break;
2496                 case PCA_BAD_QACCESS_VF:
2497                         estr = "PCA_BAD_QACCESS_VF"; break;
2498                 case PCA_MALICIOUS_REQ:
2499                         estr = "PCA_MALICIOUS_REQ"; break;
2500                 case PCA_POISONED_TLP:
2501                         estr = "PCA_POISONED_TLP"; break;
2502                 case PCA_TLP_ABORT:
2503                         estr = "PCA_TLP_ABORT"; break;
2504                 default:
2505                         goto error;
2506                 }
2507                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2508                         estr, fault.func ? "VF" : "PF", fault.func,
2509                         fault.address, fault.specinfo);
2510         }
2511
2512         /* Process THI fault */
2513         if (eicr & FM10K_EICR_THI_FAULT) {
2514                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2515                 if (err)
2516                         goto error;
2517                 switch (fault.type) {
2518                 case THI_NO_FAULT:
2519                         estr = "THI_NO_FAULT"; break;
2520                 case THI_MAL_DIS_Q_FAULT:
2521                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2522                 default:
2523                         goto error;
2524                 }
2525                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2526                         estr, fault.func ? "VF" : "PF", fault.func,
2527                         fault.address, fault.specinfo);
2528         }
2529
2530         /* Process FUM fault */
2531         if (eicr & FM10K_EICR_FUM_FAULT) {
2532                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2533                 if (err)
2534                         goto error;
2535                 switch (fault.type) {
2536                 case FUM_NO_FAULT:
2537                         estr = "FUM_NO_FAULT"; break;
2538                 case FUM_UNMAPPED_ADDR:
2539                         estr = "FUM_UNMAPPED_ADDR"; break;
2540                 case FUM_POISONED_TLP:
2541                         estr = "FUM_POISONED_TLP"; break;
2542                 case FUM_BAD_VF_QACCESS:
2543                         estr = "FUM_BAD_VF_QACCESS"; break;
2544                 case FUM_ADD_DECODE_ERR:
2545                         estr = "FUM_ADD_DECODE_ERR"; break;
2546                 case FUM_RO_ERROR:
2547                         estr = "FUM_RO_ERROR"; break;
2548                 case FUM_QPRC_CRC_ERROR:
2549                         estr = "FUM_QPRC_CRC_ERROR"; break;
2550                 case FUM_CSR_TIMEOUT:
2551                         estr = "FUM_CSR_TIMEOUT"; break;
2552                 case FUM_INVALID_TYPE:
2553                         estr = "FUM_INVALID_TYPE"; break;
2554                 case FUM_INVALID_LENGTH:
2555                         estr = "FUM_INVALID_LENGTH"; break;
2556                 case FUM_INVALID_BE:
2557                         estr = "FUM_INVALID_BE"; break;
2558                 case FUM_INVALID_ALIGN:
2559                         estr = "FUM_INVALID_ALIGN"; break;
2560                 default:
2561                         goto error;
2562                 }
2563                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2564                         estr, fault.func ? "VF" : "PF", fault.func,
2565                         fault.address, fault.specinfo);
2566         }
2567
2568         return 0;
2569 error:
2570         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2571         return err;
2572 }
2573
2574 /**
2575  * PF interrupt handler triggered by NIC for handling specific interrupt.
2576  *
2577  * @param handle
2578  *  Pointer to interrupt handle.
2579  * @param param
2580  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2581  *
2582  * @return
2583  *  void
2584  */
2585 static void
2586 fm10k_dev_interrupt_handler_pf(void *param)
2587 {
2588         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2589         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2590         uint32_t cause, status;
2591         struct fm10k_dev_info *dev_info =
2592                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2593         int status_mbx;
2594         s32 err;
2595
2596         if (hw->mac.type != fm10k_mac_pf)
2597                 return;
2598
2599         cause = FM10K_READ_REG(hw, FM10K_EICR);
2600
2601         /* Handle PCI fault cases */
2602         if (cause & FM10K_EICR_FAULT_MASK) {
2603                 PMD_INIT_LOG(ERR, "INT: find fault!");
2604                 fm10k_dev_handle_fault(hw, cause);
2605         }
2606
2607         /* Handle switch up/down */
2608         if (cause & FM10K_EICR_SWITCHNOTREADY)
2609                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2610
2611         if (cause & FM10K_EICR_SWITCHREADY) {
2612                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2613                 if (dev_info->sm_down == 1) {
2614                         fm10k_mbx_lock(hw);
2615
2616                         /* For recreating logical ports */
2617                         status_mbx = hw->mac.ops.update_lport_state(hw,
2618                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2619                         if (status_mbx == FM10K_SUCCESS)
2620                                 PMD_INIT_LOG(INFO,
2621                                         "INT: Recreated Logical port");
2622                         else
2623                                 PMD_INIT_LOG(INFO,
2624                                         "INT: Logical ports weren't recreated");
2625
2626                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2627                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2628                         if (status_mbx != FM10K_SUCCESS)
2629                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2630
2631                         fm10k_mbx_unlock(hw);
2632
2633                         /* first clear the internal SW recording structure */
2634                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2635                                                 ETH_MQ_RX_VMDQ_FLAG))
2636                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2637                                         false);
2638
2639                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2640                                         MAIN_VSI_POOL_NUMBER);
2641
2642                         /*
2643                          * Add default mac address and vlan for the logical
2644                          * ports that have been created, leave to the
2645                          * application to fully recover Rx filtering.
2646                          */
2647                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2648                                         MAIN_VSI_POOL_NUMBER);
2649
2650                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2651                                                 ETH_MQ_RX_VMDQ_FLAG))
2652                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2653                                         true);
2654
2655                         dev_info->sm_down = 0;
2656                         _rte_eth_dev_callback_process(dev,
2657                                         RTE_ETH_EVENT_INTR_LSC,
2658                                         NULL);
2659                 }
2660         }
2661
2662         /* Handle mailbox message */
2663         fm10k_mbx_lock(hw);
2664         err = hw->mbx.ops.process(hw, &hw->mbx);
2665         fm10k_mbx_unlock(hw);
2666
2667         if (err == FM10K_ERR_RESET_REQUESTED) {
2668                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2669                 dev_info->sm_down = 1;
2670                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2671                                 NULL);
2672         }
2673
2674         /* Handle SRAM error */
2675         if (cause & FM10K_EICR_SRAMERROR) {
2676                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2677
2678                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2679                 /* Write to clear pending bits */
2680                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2681
2682                 /* Todo: print out error message after shared code  updates */
2683         }
2684
2685         /* Clear these 3 events if having any */
2686         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2687                  FM10K_EICR_SWITCHREADY;
2688         if (cause)
2689                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2690
2691         /* Re-enable interrupt from device side */
2692         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2693                                         FM10K_ITR_MASK_CLEAR);
2694         /* Re-enable interrupt from host side */
2695         rte_intr_ack(dev->intr_handle);
2696 }
2697
2698 /**
2699  * VF interrupt handler triggered by NIC for handling specific interrupt.
2700  *
2701  * @param handle
2702  *  Pointer to interrupt handle.
2703  * @param param
2704  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2705  *
2706  * @return
2707  *  void
2708  */
2709 static void
2710 fm10k_dev_interrupt_handler_vf(void *param)
2711 {
2712         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2713         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2714         struct fm10k_mbx_info *mbx = &hw->mbx;
2715         struct fm10k_dev_info *dev_info =
2716                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2717         const enum fm10k_mbx_state state = mbx->state;
2718         int status_mbx;
2719
2720         if (hw->mac.type != fm10k_mac_vf)
2721                 return;
2722
2723         /* Handle mailbox message if lock is acquired */
2724         fm10k_mbx_lock(hw);
2725         hw->mbx.ops.process(hw, &hw->mbx);
2726         fm10k_mbx_unlock(hw);
2727
2728         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2729                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2730
2731                 fm10k_mbx_lock(hw);
2732                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2733                                 MAX_LPORT_NUM, 1);
2734                 fm10k_mbx_unlock(hw);
2735
2736                 /* Setting reset flag */
2737                 dev_info->sm_down = 1;
2738                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2739                                 NULL);
2740         }
2741
2742         if (dev_info->sm_down == 1 &&
2743                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2744                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2745                 fm10k_mbx_lock(hw);
2746                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2747                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2748                 if (status_mbx != FM10K_SUCCESS)
2749                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2750                 fm10k_mbx_unlock(hw);
2751
2752                 /* first clear the internal SW recording structure */
2753                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2754                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2755                                 MAIN_VSI_POOL_NUMBER);
2756
2757                 /*
2758                  * Add default mac address and vlan for the logical ports that
2759                  * have been created, leave to the application to fully recover
2760                  * Rx filtering.
2761                  */
2762                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2763                                 MAIN_VSI_POOL_NUMBER);
2764                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2765
2766                 dev_info->sm_down = 0;
2767                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2768                                 NULL);
2769         }
2770
2771         /* Re-enable interrupt from device side */
2772         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2773                                         FM10K_ITR_MASK_CLEAR);
2774         /* Re-enable interrupt from host side */
2775         rte_intr_ack(dev->intr_handle);
2776 }
2777
2778 /* Mailbox message handler in VF */
2779 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2780         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2781         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2782         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2783         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2784 };
2785
2786 static int
2787 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2788 {
2789         int err = 0;
2790
2791         /* Initialize mailbox lock */
2792         fm10k_mbx_initlock(hw);
2793
2794         /* Replace default message handler with new ones */
2795         if (hw->mac.type == fm10k_mac_vf)
2796                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2797
2798         if (err) {
2799                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2800                                 err);
2801                 return err;
2802         }
2803         /* Connect to SM for PF device or PF for VF device */
2804         return hw->mbx.ops.connect(hw, &hw->mbx);
2805 }
2806
2807 static void
2808 fm10k_close_mbx_service(struct fm10k_hw *hw)
2809 {
2810         /* Disconnect from SM for PF device or PF for VF device */
2811         hw->mbx.ops.disconnect(hw, &hw->mbx);
2812 }
2813
2814 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2815         .dev_configure          = fm10k_dev_configure,
2816         .dev_start              = fm10k_dev_start,
2817         .dev_stop               = fm10k_dev_stop,
2818         .dev_close              = fm10k_dev_close,
2819         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2820         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2821         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2822         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2823         .stats_get              = fm10k_stats_get,
2824         .xstats_get             = fm10k_xstats_get,
2825         .xstats_get_names       = fm10k_xstats_get_names,
2826         .stats_reset            = fm10k_stats_reset,
2827         .xstats_reset           = fm10k_stats_reset,
2828         .link_update            = fm10k_link_update,
2829         .dev_infos_get          = fm10k_dev_infos_get,
2830         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2831         .vlan_filter_set        = fm10k_vlan_filter_set,
2832         .vlan_offload_set       = fm10k_vlan_offload_set,
2833         .mac_addr_add           = fm10k_macaddr_add,
2834         .mac_addr_remove        = fm10k_macaddr_remove,
2835         .rx_queue_start         = fm10k_dev_rx_queue_start,
2836         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2837         .tx_queue_start         = fm10k_dev_tx_queue_start,
2838         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2839         .rx_queue_setup         = fm10k_rx_queue_setup,
2840         .rx_queue_release       = fm10k_rx_queue_release,
2841         .tx_queue_setup         = fm10k_tx_queue_setup,
2842         .tx_queue_release       = fm10k_tx_queue_release,
2843         .rx_queue_count         = fm10k_dev_rx_queue_count,
2844         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2845         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2846         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2847         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2848         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2849         .reta_update            = fm10k_reta_update,
2850         .reta_query             = fm10k_reta_query,
2851         .rss_hash_update        = fm10k_rss_hash_update,
2852         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2853 };
2854
2855 static int ftag_check_handler(__rte_unused const char *key,
2856                 const char *value, __rte_unused void *opaque)
2857 {
2858         if (strcmp(value, "1"))
2859                 return -1;
2860
2861         return 0;
2862 }
2863
2864 static int
2865 fm10k_check_ftag(struct rte_devargs *devargs)
2866 {
2867         struct rte_kvargs *kvlist;
2868         const char *ftag_key = "enable_ftag";
2869
2870         if (devargs == NULL)
2871                 return 0;
2872
2873         kvlist = rte_kvargs_parse(devargs->args, NULL);
2874         if (kvlist == NULL)
2875                 return 0;
2876
2877         if (!rte_kvargs_count(kvlist, ftag_key)) {
2878                 rte_kvargs_free(kvlist);
2879                 return 0;
2880         }
2881         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2882         if (rte_kvargs_process(kvlist, ftag_key,
2883                                 ftag_check_handler, NULL) < 0) {
2884                 rte_kvargs_free(kvlist);
2885                 return 0;
2886         }
2887         rte_kvargs_free(kvlist);
2888
2889         return 1;
2890 }
2891
2892 static uint16_t
2893 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2894                     uint16_t nb_pkts)
2895 {
2896         uint16_t nb_tx = 0;
2897         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2898
2899         while (nb_pkts) {
2900                 uint16_t ret, num;
2901
2902                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2903                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2904                                                  num);
2905                 nb_tx += ret;
2906                 nb_pkts -= ret;
2907                 if (ret < num)
2908                         break;
2909         }
2910
2911         return nb_tx;
2912 }
2913
2914 static void __attribute__((cold))
2915 fm10k_set_tx_function(struct rte_eth_dev *dev)
2916 {
2917         struct fm10k_tx_queue *txq;
2918         int i;
2919         int use_sse = 1;
2920         uint16_t tx_ftag_en = 0;
2921
2922         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2923                 /* primary process has set the ftag flag and offloads */
2924                 txq = dev->data->tx_queues[0];
2925                 if (fm10k_tx_vec_condition_check(txq)) {
2926                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2927                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2928                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2929                 } else {
2930                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2931                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2932                         dev->tx_pkt_prepare = NULL;
2933                 }
2934                 return;
2935         }
2936
2937         if (fm10k_check_ftag(dev->device->devargs))
2938                 tx_ftag_en = 1;
2939
2940         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2941                 txq = dev->data->tx_queues[i];
2942                 txq->tx_ftag_en = tx_ftag_en;
2943                 /* Check if Vector Tx is satisfied */
2944                 if (fm10k_tx_vec_condition_check(txq))
2945                         use_sse = 0;
2946         }
2947
2948         if (use_sse) {
2949                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2950                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2951                         txq = dev->data->tx_queues[i];
2952                         fm10k_txq_vec_setup(txq);
2953                 }
2954                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2955                 dev->tx_pkt_prepare = NULL;
2956         } else {
2957                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2958                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2959                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2960         }
2961 }
2962
2963 static void __attribute__((cold))
2964 fm10k_set_rx_function(struct rte_eth_dev *dev)
2965 {
2966         struct fm10k_dev_info *dev_info =
2967                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2968         uint16_t i, rx_using_sse;
2969         uint16_t rx_ftag_en = 0;
2970
2971         if (fm10k_check_ftag(dev->device->devargs))
2972                 rx_ftag_en = 1;
2973
2974         /* In order to allow Vector Rx there are a few configuration
2975          * conditions to be met.
2976          */
2977         if (!fm10k_rx_vec_condition_check(dev) &&
2978                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2979                 if (dev->data->scattered_rx)
2980                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2981                 else
2982                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2983         } else if (dev->data->scattered_rx)
2984                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2985         else
2986                 dev->rx_pkt_burst = fm10k_recv_pkts;
2987
2988         rx_using_sse =
2989                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2990                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2991
2992         if (rx_using_sse)
2993                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2994         else
2995                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2996
2997         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2998                 return;
2999
3000         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3001                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3002
3003                 rxq->rx_using_sse = rx_using_sse;
3004                 rxq->rx_ftag_en = rx_ftag_en;
3005         }
3006 }
3007
3008 static void
3009 fm10k_params_init(struct rte_eth_dev *dev)
3010 {
3011         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3012         struct fm10k_dev_info *info =
3013                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3014
3015         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3016          * there is no way to get link status without reading BAR4.  Until this
3017          * works, assume we have maximum bandwidth.
3018          * @todo - fix bus info
3019          */
3020         hw->bus_caps.speed = fm10k_bus_speed_8000;
3021         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3022         hw->bus_caps.payload = fm10k_bus_payload_512;
3023         hw->bus.speed = fm10k_bus_speed_8000;
3024         hw->bus.width = fm10k_bus_width_pcie_x8;
3025         hw->bus.payload = fm10k_bus_payload_256;
3026
3027         info->rx_vec_allowed = true;
3028         info->sm_down = false;
3029 }
3030
3031 static int
3032 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3033 {
3034         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3035         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3036         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3037         int diag, i;
3038         struct fm10k_macvlan_filter_info *macvlan;
3039
3040         PMD_INIT_FUNC_TRACE();
3041
3042         dev->dev_ops = &fm10k_eth_dev_ops;
3043         dev->rx_pkt_burst = &fm10k_recv_pkts;
3044         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3045         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3046
3047         /*
3048          * Primary process does the whole initialization, for secondary
3049          * processes, we just select the same Rx and Tx function as primary.
3050          */
3051         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3052                 fm10k_set_rx_function(dev);
3053                 fm10k_set_tx_function(dev);
3054                 return 0;
3055         }
3056
3057         rte_eth_copy_pci_info(dev, pdev);
3058
3059         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3060         memset(macvlan, 0, sizeof(*macvlan));
3061         /* Vendor and Device ID need to be set before init of shared code */
3062         memset(hw, 0, sizeof(*hw));
3063         hw->device_id = pdev->id.device_id;
3064         hw->vendor_id = pdev->id.vendor_id;
3065         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3066         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3067         hw->revision_id = 0;
3068         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3069         if (hw->hw_addr == NULL) {
3070                 PMD_INIT_LOG(ERR, "Bad mem resource."
3071                         " Try to blacklist unused devices.");
3072                 return -EIO;
3073         }
3074
3075         /* Store fm10k_adapter pointer */
3076         hw->back = dev->data->dev_private;
3077
3078         /* Initialize the shared code */
3079         diag = fm10k_init_shared_code(hw);
3080         if (diag != FM10K_SUCCESS) {
3081                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3082                 return -EIO;
3083         }
3084
3085         /* Initialize parameters */
3086         fm10k_params_init(dev);
3087
3088         /* Initialize the hw */
3089         diag = fm10k_init_hw(hw);
3090         if (diag != FM10K_SUCCESS) {
3091                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3092                 return -EIO;
3093         }
3094
3095         /* Initialize MAC address(es) */
3096         dev->data->mac_addrs = rte_zmalloc("fm10k",
3097                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3098         if (dev->data->mac_addrs == NULL) {
3099                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3100                 return -ENOMEM;
3101         }
3102
3103         diag = fm10k_read_mac_addr(hw);
3104
3105         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3106                         &dev->data->mac_addrs[0]);
3107
3108         if (diag != FM10K_SUCCESS ||
3109                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3110
3111                 /* Generate a random addr */
3112                 rte_eth_random_addr(hw->mac.addr);
3113                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3114                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3115                 &dev->data->mac_addrs[0]);
3116         }
3117
3118         /* Reset the hw statistics */
3119         diag = fm10k_stats_reset(dev);
3120         if (diag != 0) {
3121                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3122                 return diag;
3123         }
3124
3125         /* Reset the hw */
3126         diag = fm10k_reset_hw(hw);
3127         if (diag != FM10K_SUCCESS) {
3128                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3129                 return -EIO;
3130         }
3131
3132         /* Setup mailbox service */
3133         diag = fm10k_setup_mbx_service(hw);
3134         if (diag != FM10K_SUCCESS) {
3135                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3136                 return -EIO;
3137         }
3138
3139         /*PF/VF has different interrupt handling mechanism */
3140         if (hw->mac.type == fm10k_mac_pf) {
3141                 /* register callback func to eal lib */
3142                 rte_intr_callback_register(intr_handle,
3143                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3144
3145                 /* enable MISC interrupt */
3146                 fm10k_dev_enable_intr_pf(dev);
3147         } else { /* VF */
3148                 rte_intr_callback_register(intr_handle,
3149                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3150
3151                 fm10k_dev_enable_intr_vf(dev);
3152         }
3153
3154         /* Enable intr after callback registered */
3155         rte_intr_enable(intr_handle);
3156
3157         hw->mac.ops.update_int_moderator(hw);
3158
3159         /* Make sure Switch Manager is ready before going forward. */
3160         if (hw->mac.type == fm10k_mac_pf) {
3161                 int switch_ready = 0;
3162
3163                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3164                         fm10k_mbx_lock(hw);
3165                         hw->mac.ops.get_host_state(hw, &switch_ready);
3166                         fm10k_mbx_unlock(hw);
3167                         if (switch_ready)
3168                                 break;
3169                         /* Delay some time to acquire async LPORT_MAP info. */
3170                         rte_delay_us(WAIT_SWITCH_MSG_US);
3171                 }
3172
3173                 if (switch_ready == 0) {
3174                         PMD_INIT_LOG(ERR, "switch is not ready");
3175                         return -1;
3176                 }
3177         }
3178
3179         /*
3180          * Below function will trigger operations on mailbox, acquire lock to
3181          * avoid race condition from interrupt handler. Operations on mailbox
3182          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3183          * will handle and generate an interrupt to our side. Then,  FIFO in
3184          * mailbox will be touched.
3185          */
3186         fm10k_mbx_lock(hw);
3187         /* Enable port first */
3188         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3189                                         MAX_LPORT_NUM, 1);
3190
3191         /* Set unicast mode by default. App can change to other mode in other
3192          * API func.
3193          */
3194         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3195                                         FM10K_XCAST_MODE_NONE);
3196
3197         fm10k_mbx_unlock(hw);
3198
3199         /* Make sure default VID is ready before going forward. */
3200         if (hw->mac.type == fm10k_mac_pf) {
3201                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3202                         if (hw->mac.default_vid)
3203                                 break;
3204                         /* Delay some time to acquire async port VLAN info. */
3205                         rte_delay_us(WAIT_SWITCH_MSG_US);
3206                 }
3207
3208                 if (!hw->mac.default_vid) {
3209                         PMD_INIT_LOG(ERR, "default VID is not ready");
3210                         return -1;
3211                 }
3212         }
3213
3214         /* Add default mac address */
3215         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3216                 MAIN_VSI_POOL_NUMBER);
3217
3218         return 0;
3219 }
3220
3221 static int
3222 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3223 {
3224         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3225         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3226         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3227         PMD_INIT_FUNC_TRACE();
3228
3229         /* only uninitialize in the primary process */
3230         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3231                 return 0;
3232
3233         /* safe to close dev here */
3234         fm10k_dev_close(dev);
3235
3236         dev->dev_ops = NULL;
3237         dev->rx_pkt_burst = NULL;
3238         dev->tx_pkt_burst = NULL;
3239
3240         /* disable uio/vfio intr */
3241         rte_intr_disable(intr_handle);
3242
3243         /*PF/VF has different interrupt handling mechanism */
3244         if (hw->mac.type == fm10k_mac_pf) {
3245                 /* disable interrupt */
3246                 fm10k_dev_disable_intr_pf(dev);
3247
3248                 /* unregister callback func to eal lib */
3249                 rte_intr_callback_unregister(intr_handle,
3250                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3251         } else {
3252                 /* disable interrupt */
3253                 fm10k_dev_disable_intr_vf(dev);
3254
3255                 rte_intr_callback_unregister(intr_handle,
3256                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3257         }
3258
3259         return 0;
3260 }
3261
3262 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3263         struct rte_pci_device *pci_dev)
3264 {
3265         return rte_eth_dev_pci_generic_probe(pci_dev,
3266                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3267 }
3268
3269 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3270 {
3271         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3272 }
3273
3274 /*
3275  * The set of PCI devices this driver supports. This driver will enable both PF
3276  * and SRIOV-VF devices.
3277  */
3278 static const struct rte_pci_id pci_id_fm10k_map[] = {
3279         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3280         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3281         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3282         { .vendor_id = 0, /* sentinel */ },
3283 };
3284
3285 static struct rte_pci_driver rte_pmd_fm10k = {
3286         .id_table = pci_id_fm10k_map,
3287         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3288         .probe = eth_fm10k_pci_probe,
3289         .remove = eth_fm10k_pci_remove,
3290 };
3291
3292 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3293 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3294 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3295
3296 RTE_INIT(fm10k_init_log)
3297 {
3298         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3299         if (fm10k_logtype_init >= 0)
3300                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3301         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3302         if (fm10k_logtype_driver >= 0)
3303                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3304 }