drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 struct fm10k_xstats_name_off {
  68         char name[RTE_ETH_XSTATS_NAME_SIZE];
  69         unsigned offset;
  70 };
  71
  72 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  73         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  74         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  75         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  76         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  77         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  78         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  79         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  80         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  81                 nodesc_drop)},
  82 };
  83
  84 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  85                 sizeof(fm10k_hw_stats_strings[0]))
  86
  87 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  88         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  89         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  90         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  91 };
  92
  93 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  94                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  95
  96 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
  97         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
  98         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
  99 };
 100
 101 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 102                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 103
 104 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 105                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 106 static int
 107 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 108
 109 static void
 110 fm10k_mbx_initlock(struct fm10k_hw *hw)
 111 {
 112         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 113 }
 114
 115 static void
 116 fm10k_mbx_lock(struct fm10k_hw *hw)
 117 {
 118         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 119                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 120 }
 121
 122 static void
 123 fm10k_mbx_unlock(struct fm10k_hw *hw)
 124 {
 125         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 126 }
 127
 128 /* Stubs needed for linkage when vPMD is disabled */
 129 int __attribute__((weak))
 130 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 131 {
 132         return -1;
 133 }
 134
 135 uint16_t __attribute__((weak))
 136 fm10k_recv_pkts_vec(
 137         __rte_unused void *rx_queue,
 138         __rte_unused struct rte_mbuf **rx_pkts,
 139         __rte_unused uint16_t nb_pkts)
 140 {
 141         return 0;
 142 }
 143
 144 uint16_t __attribute__((weak))
 145 fm10k_recv_scattered_pkts_vec(
 146                 __rte_unused void *rx_queue,
 147                 __rte_unused struct rte_mbuf **rx_pkts,
 148                 __rte_unused uint16_t nb_pkts)
 149 {
 150         return 0;
 151 }
 152
 153 int __attribute__((weak))
 154 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 155
 156 {
 157         return -1;
 158 }
 159
 160 void __attribute__((weak))
 161 fm10k_rx_queue_release_mbufs_vec(
 162                 __rte_unused struct fm10k_rx_queue *rxq)
 163 {
 164         return;
 165 }
 166
 167 void __attribute__((weak))
 168 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 169 {
 170         return;
 171 }
 172
 173 int __attribute__((weak))
 174 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 175 {
 176         return -1;
 177 }
 178
 179 uint16_t __attribute__((weak))
 180 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 181                            __rte_unused struct rte_mbuf **tx_pkts,
 182                            __rte_unused uint16_t nb_pkts)
 183 {
 184         return 0;
 185 }
 186
 187 /*
 188  * reset queue to initial state, allocate software buffers used when starting
 189  * device.
 190  * return 0 on success
 191  * return -ENOMEM if buffers cannot be allocated
 192  * return -EINVAL if buffers do not satisfy alignment condition
 193  */
 194 static inline int
 195 rx_queue_reset(struct fm10k_rx_queue *q)
 196 {
 197         static const union fm10k_rx_desc zero = {{0} };
 198         uint64_t dma_addr;
 199         int i, diag;
 200         PMD_INIT_FUNC_TRACE();
 201
 202         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 203         if (diag != 0)
 204                 return -ENOMEM;
 205
 206         for (i = 0; i < q->nb_desc; ++i) {
 207                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 208                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 209                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 210                                                 q->nb_desc);
 211                         return -EINVAL;
 212                 }
 213                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 214                 q->hw_ring[i].q.pkt_addr = dma_addr;
 215                 q->hw_ring[i].q.hdr_addr = dma_addr;
 216         }
 217
 218         /* initialize extra software ring entries. Space for these extra
 219          * entries is always allocated.
 220          */
 221         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 222         for (i = 0; i < q->nb_fake_desc; ++i) {
 223                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 224                 q->hw_ring[q->nb_desc + i] = zero;
 225         }
 226
 227         q->next_dd = 0;
 228         q->next_alloc = 0;
 229         q->next_trigger = q->alloc_thresh - 1;
 230         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 231         q->rxrearm_start = 0;
 232         q->rxrearm_nb = 0;
 233
 234         return 0;
 235 }
 236
 237 /*
 238  * clean queue, descriptor rings, free software buffers used when stopping
 239  * device.
 240  */
 241 static inline void
 242 rx_queue_clean(struct fm10k_rx_queue *q)
 243 {
 244         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 245         uint32_t i;
 246         PMD_INIT_FUNC_TRACE();
 247
 248         /* zero descriptor rings */
 249         for (i = 0; i < q->nb_desc; ++i)
 250                 q->hw_ring[i] = zero;
 251
 252         /* zero faked descriptors */
 253         for (i = 0; i < q->nb_fake_desc; ++i)
 254                 q->hw_ring[q->nb_desc + i] = zero;
 255
 256         /* vPMD driver has a different way of releasing mbufs. */
 257         if (q->rx_using_sse) {
 258                 fm10k_rx_queue_release_mbufs_vec(q);
 259                 return;
 260         }
 261
 262         /* free software buffers */
 263         for (i = 0; i < q->nb_desc; ++i) {
 264                 if (q->sw_ring[i]) {
 265                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 266                         q->sw_ring[i] = NULL;
 267                 }
 268         }
 269 }
 270
 271 /*
 272  * free all queue memory used when releasing the queue (i.e. configure)
 273  */
 274 static inline void
 275 rx_queue_free(struct fm10k_rx_queue *q)
 276 {
 277         PMD_INIT_FUNC_TRACE();
 278         if (q) {
 279                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 280                 rx_queue_clean(q);
 281                 if (q->sw_ring) {
 282                         rte_free(q->sw_ring);
 283                         q->sw_ring = NULL;
 284                 }
 285                 rte_free(q);
 286                 q = NULL;
 287         }
 288 }
 289
 290 /*
 291  * disable RX queue, wait unitl HW finished necessary flush operation
 292  */
 293 static inline int
 294 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 295 {
 296         uint32_t reg, i;
 297
 298         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 299         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 300                         reg & ~FM10K_RXQCTL_ENABLE);
 301
 302         /* Wait 100us at most */
 303         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 304                 rte_delay_us(1);
 305                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 306                 if (!(reg & FM10K_RXQCTL_ENABLE))
 307                         break;
 308         }
 309
 310         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 311                 return -1;
 312
 313         return 0;
 314 }
 315
 316 /*
 317  * reset queue to initial state, allocate software buffers used when starting
 318  * device
 319  */
 320 static inline void
 321 tx_queue_reset(struct fm10k_tx_queue *q)
 322 {
 323         PMD_INIT_FUNC_TRACE();
 324         q->last_free = 0;
 325         q->next_free = 0;
 326         q->nb_used = 0;
 327         q->nb_free = q->nb_desc - 1;
 328         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 329         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 330 }
 331
 332 /*
 333  * clean queue, descriptor rings, free software buffers used when stopping
 334  * device
 335  */
 336 static inline void
 337 tx_queue_clean(struct fm10k_tx_queue *q)
 338 {
 339         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 340         uint32_t i;
 341         PMD_INIT_FUNC_TRACE();
 342
 343         /* zero descriptor rings */
 344         for (i = 0; i < q->nb_desc; ++i)
 345                 q->hw_ring[i] = zero;
 346
 347         /* free software buffers */
 348         for (i = 0; i < q->nb_desc; ++i) {
 349                 if (q->sw_ring[i]) {
 350                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 351                         q->sw_ring[i] = NULL;
 352                 }
 353         }
 354 }
 355
 356 /*
 357  * free all queue memory used when releasing the queue (i.e. configure)
 358  */
 359 static inline void
 360 tx_queue_free(struct fm10k_tx_queue *q)
 361 {
 362         PMD_INIT_FUNC_TRACE();
 363         if (q) {
 364                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 365                 tx_queue_clean(q);
 366                 if (q->rs_tracker.list) {
 367                         rte_free(q->rs_tracker.list);
 368                         q->rs_tracker.list = NULL;
 369                 }
 370                 if (q->sw_ring) {
 371                         rte_free(q->sw_ring);
 372                         q->sw_ring = NULL;
 373                 }
 374                 rte_free(q);
 375                 q = NULL;
 376         }
 377 }
 378
 379 /*
 380  * disable TX queue, wait unitl HW finished necessary flush operation
 381  */
 382 static inline int
 383 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 384 {
 385         uint32_t reg, i;
 386
 387         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 388         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 389                         reg & ~FM10K_TXDCTL_ENABLE);
 390
 391         /* Wait 100us at most */
 392         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 393                 rte_delay_us(1);
 394                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 395                 if (!(reg & FM10K_TXDCTL_ENABLE))
 396                         break;
 397         }
 398
 399         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 400                 return -1;
 401
 402         return 0;
 403 }
 404
 405 static int
 406 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 407 {
 408         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 409         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 410         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 411         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 412
 413         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 414
 415         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 416                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 417                 return -EINVAL;
 418         }
 419
 420         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 421                 return 0;
 422
 423         if (hw->mac.type == fm10k_mac_vf) {
 424                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 425                 return -EINVAL;
 426         }
 427
 428         /* Check VMDQ queue pool number */
 429         if (vmdq_conf->nb_queue_pools >
 430                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 431                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 432                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 433                         vmdq_conf->nb_queue_pools);
 434                 return -EINVAL;
 435         }
 436
 437         return 0;
 438 }
 439
 440 static const struct fm10k_txq_ops def_txq_ops = {
 441         .reset = tx_queue_reset,
 442 };
 443
 444 static int
 445 fm10k_dev_configure(struct rte_eth_dev *dev)
 446 {
 447         int ret;
 448         struct rte_eth_dev_info dev_info;
 449         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
 450
 451         PMD_INIT_FUNC_TRACE();
 452
 453         if ((rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP) == 0)
 454                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 455
 456         fm10k_dev_infos_get(dev, &dev_info);
 457         if ((rx_offloads & dev_info.rx_offload_capa) != rx_offloads) {
 458                 PMD_DRV_LOG(ERR, "Some Rx offloads are not supported "
 459                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 460                             rx_offloads, dev_info.rx_offload_capa);
 461                 return -ENOTSUP;
 462         }
 463         /* multipe queue mode checking */
 464         ret  = fm10k_check_mq_mode(dev);
 465         if (ret != 0) {
 466                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 467                             ret);
 468                 return ret;
 469         }
 470
 471         dev->data->scattered_rx = 0;
 472
 473         return 0;
 474 }
 475
 476 /* fls = find last set bit = 32 minus the number of leading zeros */
 477 #ifndef fls
 478 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 479 #endif
 480
 481 static void
 482 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 483 {
 484         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 485         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 486         uint32_t i;
 487
 488         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 489
 490         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 491                 if (!vmdq_conf->pool_map[i].pools)
 492                         continue;
 493                 fm10k_mbx_lock(hw);
 494                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 495                 fm10k_mbx_unlock(hw);
 496         }
 497 }
 498
 499 static void
 500 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 501 {
 502         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 503
 504         /* Add default mac address */
 505         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 506                 MAIN_VSI_POOL_NUMBER);
 507 }
 508
 509 static void
 510 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 511 {
 512         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 513         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 514         uint32_t mrqc, *key, i, reta, j;
 515         uint64_t hf;
 516
 517 #define RSS_KEY_SIZE 40
 518         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 519                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 520                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 521                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 522                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 523                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 524         };
 525
 526         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 527                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 528                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 529                 return;
 530         }
 531
 532         /* random key is rss_intel_key (default) or user provided (rss_key) */
 533         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 534                 key = (uint32_t *)rss_intel_key;
 535         else
 536                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 537
 538         /* Now fill our hash function seeds, 4 bytes at a time */
 539         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 540                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 541
 542         /*
 543          * Fill in redirection table
 544          * The byte-swap is needed because NIC registers are in
 545          * little-endian order.
 546          */
 547         reta = 0;
 548         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 549                 if (j == dev->data->nb_rx_queues)
 550                         j = 0;
 551                 reta = (reta << CHAR_BIT) | j;
 552                 if ((i & 3) == 3)
 553                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 554                                         rte_bswap32(reta));
 555         }
 556
 557         /*
 558          * Generate RSS hash based on packet types, TCP/UDP
 559          * port numbers and/or IPv4/v6 src and dst addresses
 560          */
 561         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 562         mrqc = 0;
 563         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 564         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 565         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 566         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 567         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 568         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 569         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 570         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 571         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 572
 573         if (mrqc == 0) {
 574                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 575                         "supported", hf);
 576                 return;
 577         }
 578
 579         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 580 }
 581
 582 static void
 583 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 584 {
 585         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 586         uint32_t i;
 587
 588         for (i = 0; i < nb_lport_new; i++) {
 589                 /* Set unicast mode by default. App can change
 590                  * to other mode in other API func.
 591                  */
 592                 fm10k_mbx_lock(hw);
 593                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 594                         FM10K_XCAST_MODE_NONE);
 595                 fm10k_mbx_unlock(hw);
 596         }
 597 }
 598
 599 static void
 600 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 601 {
 602         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 603         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 604         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 605         struct fm10k_macvlan_filter_info *macvlan;
 606         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 607         uint16_t nb_lport_new;
 608
 609         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 610         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 611
 612         fm10k_dev_rss_configure(dev);
 613
 614         /* only PF supports VMDQ */
 615         if (hw->mac.type != fm10k_mac_pf)
 616                 return;
 617
 618         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 619                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 620
 621         /* no pool number change, no need to update logic port and VLAN/MAC */
 622         if (macvlan->nb_queue_pools == nb_queue_pools)
 623                 return;
 624
 625         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 626         fm10k_dev_logic_port_update(dev, nb_lport_new);
 627
 628         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 629         memset(dev->data->mac_addrs, 0,
 630                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 631         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 632                 &dev->data->mac_addrs[0]);
 633         memset(macvlan, 0, sizeof(*macvlan));
 634         macvlan->nb_queue_pools = nb_queue_pools;
 635
 636         if (nb_queue_pools)
 637                 fm10k_dev_vmdq_rx_configure(dev);
 638         else
 639                 fm10k_dev_pf_main_vsi_reset(dev);
 640 }
 641
 642 static int
 643 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 644 {
 645         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 646         int i, ret;
 647         struct fm10k_tx_queue *txq;
 648         uint64_t base_addr;
 649         uint32_t size;
 650
 651         /* Disable TXINT to avoid possible interrupt */
 652         for (i = 0; i < hw->mac.max_queues; i++)
 653                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 654                                 3 << FM10K_TXINT_TIMER_SHIFT);
 655
 656         /* Setup TX queue */
 657         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 658                 txq = dev->data->tx_queues[i];
 659                 base_addr = txq->hw_ring_phys_addr;
 660                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 661
 662                 /* disable queue to avoid issues while updating state */
 663                 ret = tx_queue_disable(hw, i);
 664                 if (ret) {
 665                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 666                         return -1;
 667                 }
 668                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 669                  * register is read-only for VF.
 670                  */
 671                 if (fm10k_check_ftag(dev->device->devargs)) {
 672                         if (hw->mac.type == fm10k_mac_pf) {
 673                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 674                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 675                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 676                         } else {
 677                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 678                                 return -ENOTSUP;
 679                         }
 680                 }
 681
 682                 /* set location and size for descriptor ring */
 683                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 684                                 base_addr & UINT64_LOWER_32BITS_MASK);
 685                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 686                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 687                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 688
 689                 /* assign default SGLORT for each TX queue by PF */
 690                 if (hw->mac.type == fm10k_mac_pf)
 691                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 692         }
 693
 694         /* set up vector or scalar TX function as appropriate */
 695         fm10k_set_tx_function(dev);
 696
 697         return 0;
 698 }
 699
 700 static int
 701 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 702 {
 703         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 704         struct fm10k_macvlan_filter_info *macvlan;
 705         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 706         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 707         int i, ret;
 708         struct fm10k_rx_queue *rxq;
 709         uint64_t base_addr;
 710         uint32_t size;
 711         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 712         uint32_t logic_port = hw->mac.dglort_map;
 713         uint16_t buf_size;
 714         uint16_t queue_stride = 0;
 715
 716         /* enable RXINT for interrupt mode */
 717         i = 0;
 718         if (rte_intr_dp_is_en(intr_handle)) {
 719                 for (; i < dev->data->nb_rx_queues; i++) {
 720                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 721                         if (hw->mac.type == fm10k_mac_pf)
 722                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 723                                         FM10K_ITR_AUTOMASK |
 724                                         FM10K_ITR_MASK_CLEAR);
 725                         else
 726                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 727                                         FM10K_ITR_AUTOMASK |
 728                                         FM10K_ITR_MASK_CLEAR);
 729                 }
 730         }
 731         /* Disable other RXINT to avoid possible interrupt */
 732         for (; i < hw->mac.max_queues; i++)
 733                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 734                         3 << FM10K_RXINT_TIMER_SHIFT);
 735
 736         /* Setup RX queues */
 737         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 738                 rxq = dev->data->rx_queues[i];
 739                 base_addr = rxq->hw_ring_phys_addr;
 740                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 741
 742                 /* disable queue to avoid issues while updating state */
 743                 ret = rx_queue_disable(hw, i);
 744                 if (ret) {
 745                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 746                         return -1;
 747                 }
 748
 749                 /* Setup the Base and Length of the Rx Descriptor Ring */
 750                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 751                                 base_addr & UINT64_LOWER_32BITS_MASK);
 752                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 753                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 754                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 755
 756                 /* Configure the Rx buffer size for one buff without split */
 757                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 758                         RTE_PKTMBUF_HEADROOM);
 759                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 760                  * reserved for this purpose, and the worst case could be 511B.
 761                  * But SRR reg assumes all buffers have the same size. In order
 762                  * to fill the gap, we'll have to consider the worst case and
 763                  * assume 512B is reserved. If we don't do so, it's possible
 764                  * for HW to overwrite data to next mbuf.
 765                  */
 766                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 767
 768                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 769                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 770                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 771
 772                 /* It adds dual VLAN length for supporting dual VLAN */
 773                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 774                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 775                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 776                         uint32_t reg;
 777                         dev->data->scattered_rx = 1;
 778                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 779                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 780                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 781                 }
 782
 783                 /* Enable drop on empty, it's RO for VF */
 784                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 785                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 786
 787                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 788                 FM10K_WRITE_FLUSH(hw);
 789         }
 790
 791         /* Configure VMDQ/RSS if applicable */
 792         fm10k_dev_mq_rx_configure(dev);
 793
 794         /* Decide the best RX function */
 795         fm10k_set_rx_function(dev);
 796
 797         /* update RX_SGLORT for loopback suppress*/
 798         if (hw->mac.type != fm10k_mac_pf)
 799                 return 0;
 800         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 801         if (macvlan->nb_queue_pools)
 802                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 803         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 804                 if (i && queue_stride && !(i % queue_stride))
 805                         logic_port++;
 806                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 807         }
 808
 809         return 0;
 810 }
 811
 812 static int
 813 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 814 {
 815         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 816         int err = -1;
 817         uint32_t reg;
 818         struct fm10k_rx_queue *rxq;
 819
 820         PMD_INIT_FUNC_TRACE();
 821
 822         if (rx_queue_id < dev->data->nb_rx_queues) {
 823                 rxq = dev->data->rx_queues[rx_queue_id];
 824                 err = rx_queue_reset(rxq);
 825                 if (err == -ENOMEM) {
 826                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 827                         return err;
 828                 } else if (err == -EINVAL) {
 829                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 830                                 " %d", err);
 831                         return err;
 832                 }
 833
 834                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 835                  * Note: this must be done AFTER the queue is enabled on real
 836                  * hardware, but BEFORE the queue is enabled when using the
 837                  * emulation platform. Do it in both places for now and remove
 838                  * this comment and the following two register writes when the
 839                  * emulation platform is no longer being used.
 840                  */
 841                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843
 844                 /* Set PF ownership flag for PF devices */
 845                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 846                 if (hw->mac.type == fm10k_mac_pf)
 847                         reg |= FM10K_RXQCTL_PF;
 848                 reg |= FM10K_RXQCTL_ENABLE;
 849                 /* enable RX queue */
 850                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 851                 FM10K_WRITE_FLUSH(hw);
 852
 853                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 854                  * Note: this must be done AFTER the queue is enabled
 855                  */
 856                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 857                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 858                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 859         }
 860
 861         return err;
 862 }
 863
 864 static int
 865 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 866 {
 867         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 868
 869         PMD_INIT_FUNC_TRACE();
 870
 871         if (rx_queue_id < dev->data->nb_rx_queues) {
 872                 /* Disable RX queue */
 873                 rx_queue_disable(hw, rx_queue_id);
 874
 875                 /* Free mbuf and clean HW ring */
 876                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 877                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 878         }
 879
 880         return 0;
 881 }
 882
 883 static int
 884 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 885 {
 886         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 887         /** @todo - this should be defined in the shared code */
 888 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 889         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 890         int err = 0;
 891
 892         PMD_INIT_FUNC_TRACE();
 893
 894         if (tx_queue_id < dev->data->nb_tx_queues) {
 895                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 896
 897                 q->ops->reset(q);
 898
 899                 /* reset head and tail pointers */
 900                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 901                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 902
 903                 /* enable TX queue */
 904                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 905                                         FM10K_TXDCTL_ENABLE | txdctl);
 906                 FM10K_WRITE_FLUSH(hw);
 907                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 908         } else
 909                 err = -1;
 910
 911         return err;
 912 }
 913
 914 static int
 915 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 916 {
 917         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 918
 919         PMD_INIT_FUNC_TRACE();
 920
 921         if (tx_queue_id < dev->data->nb_tx_queues) {
 922                 tx_queue_disable(hw, tx_queue_id);
 923                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 924                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 925         }
 926
 927         return 0;
 928 }
 929
 930 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 931 {
 932         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 933                 != FM10K_DGLORTMAP_NONE);
 934 }
 935
 936 static void
 937 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 938 {
 939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         int status;
 941
 942         PMD_INIT_FUNC_TRACE();
 943
 944         /* Return if it didn't acquire valid glort range */
 945         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 946                 return;
 947
 948         fm10k_mbx_lock(hw);
 949         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 950                                 FM10K_XCAST_MODE_PROMISC);
 951         fm10k_mbx_unlock(hw);
 952
 953         if (status != FM10K_SUCCESS)
 954                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 955 }
 956
 957 static void
 958 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 959 {
 960         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 961         uint8_t mode;
 962         int status;
 963
 964         PMD_INIT_FUNC_TRACE();
 965
 966         /* Return if it didn't acquire valid glort range */
 967         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 968                 return;
 969
 970         if (dev->data->all_multicast == 1)
 971                 mode = FM10K_XCAST_MODE_ALLMULTI;
 972         else
 973                 mode = FM10K_XCAST_MODE_NONE;
 974
 975         fm10k_mbx_lock(hw);
 976         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 977                                 mode);
 978         fm10k_mbx_unlock(hw);
 979
 980         if (status != FM10K_SUCCESS)
 981                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 982 }
 983
 984 static void
 985 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 986 {
 987         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 988         int status;
 989
 990         PMD_INIT_FUNC_TRACE();
 991
 992         /* Return if it didn't acquire valid glort range */
 993         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 994                 return;
 995
 996         /* If promiscuous mode is enabled, it doesn't make sense to enable
 997          * allmulticast and disable promiscuous since fm10k only can select
 998          * one of the modes.
 999          */
1000         if (dev->data->promiscuous) {
1001                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1002                         "needn't enable allmulticast");
1003                 return;
1004         }
1005
1006         fm10k_mbx_lock(hw);
1007         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1008                                 FM10K_XCAST_MODE_ALLMULTI);
1009         fm10k_mbx_unlock(hw);
1010
1011         if (status != FM10K_SUCCESS)
1012                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1013 }
1014
1015 static void
1016 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1017 {
1018         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1019         int status;
1020
1021         PMD_INIT_FUNC_TRACE();
1022
1023         /* Return if it didn't acquire valid glort range */
1024         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1025                 return;
1026
1027         if (dev->data->promiscuous) {
1028                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1029                         "since promisc mode is enabled");
1030                 return;
1031         }
1032
1033         fm10k_mbx_lock(hw);
1034         /* Change mode to unicast mode */
1035         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1036                                 FM10K_XCAST_MODE_NONE);
1037         fm10k_mbx_unlock(hw);
1038
1039         if (status != FM10K_SUCCESS)
1040                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1041 }
1042
1043 static void
1044 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1045 {
1046         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1047         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1048         uint16_t nb_queue_pools;
1049         struct fm10k_macvlan_filter_info *macvlan;
1050
1051         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1052         nb_queue_pools = macvlan->nb_queue_pools;
1053         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1054         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1055
1056         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1057         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1058         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1059                         hw->mac.dglort_map;
1060         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1061         /* Configure VMDQ/RSS DGlort Decoder */
1062         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1063
1064         /* Flow Director configurations, only queue number is valid. */
1065         dglortdec = fls(dev->data->nb_rx_queues - 1);
1066         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1067                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1068         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1069         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1070
1071         /* Invalidate all other GLORT entries */
1072         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1073                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1074                                 FM10K_DGLORTMAP_NONE);
1075 }
1076
1077 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1078 static int
1079 fm10k_dev_start(struct rte_eth_dev *dev)
1080 {
1081         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1082         int i, diag;
1083
1084         PMD_INIT_FUNC_TRACE();
1085
1086         /* stop, init, then start the hw */
1087         diag = fm10k_stop_hw(hw);
1088         if (diag != FM10K_SUCCESS) {
1089                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1090                 return -EIO;
1091         }
1092
1093         diag = fm10k_init_hw(hw);
1094         if (diag != FM10K_SUCCESS) {
1095                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1096                 return -EIO;
1097         }
1098
1099         diag = fm10k_start_hw(hw);
1100         if (diag != FM10K_SUCCESS) {
1101                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1102                 return -EIO;
1103         }
1104
1105         diag = fm10k_dev_tx_init(dev);
1106         if (diag) {
1107                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1108                 return diag;
1109         }
1110
1111         if (fm10k_dev_rxq_interrupt_setup(dev))
1112                 return -EIO;
1113
1114         diag = fm10k_dev_rx_init(dev);
1115         if (diag) {
1116                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1117                 return diag;
1118         }
1119
1120         if (hw->mac.type == fm10k_mac_pf)
1121                 fm10k_dev_dglort_map_configure(dev);
1122
1123         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1124                 struct fm10k_rx_queue *rxq;
1125                 rxq = dev->data->rx_queues[i];
1126
1127                 if (rxq->rx_deferred_start)
1128                         continue;
1129                 diag = fm10k_dev_rx_queue_start(dev, i);
1130                 if (diag != 0) {
1131                         int j;
1132                         for (j = 0; j < i; ++j)
1133                                 rx_queue_clean(dev->data->rx_queues[j]);
1134                         return diag;
1135                 }
1136         }
1137
1138         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1139                 struct fm10k_tx_queue *txq;
1140                 txq = dev->data->tx_queues[i];
1141
1142                 if (txq->tx_deferred_start)
1143                         continue;
1144                 diag = fm10k_dev_tx_queue_start(dev, i);
1145                 if (diag != 0) {
1146                         int j;
1147                         for (j = 0; j < i; ++j)
1148                                 tx_queue_clean(dev->data->tx_queues[j]);
1149                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1150                                 rx_queue_clean(dev->data->rx_queues[j]);
1151                         return diag;
1152                 }
1153         }
1154
1155         /* Update default vlan when not in VMDQ mode */
1156         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1157                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1158
1159         fm10k_link_update(dev, 0);
1160
1161         return 0;
1162 }
1163
1164 static void
1165 fm10k_dev_stop(struct rte_eth_dev *dev)
1166 {
1167         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1168         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1169         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1170         int i;
1171
1172         PMD_INIT_FUNC_TRACE();
1173
1174         if (dev->data->tx_queues)
1175                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1176                         fm10k_dev_tx_queue_stop(dev, i);
1177
1178         if (dev->data->rx_queues)
1179                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1180                         fm10k_dev_rx_queue_stop(dev, i);
1181
1182         /* Disable datapath event */
1183         if (rte_intr_dp_is_en(intr_handle)) {
1184                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1185                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1186                                 3 << FM10K_RXINT_TIMER_SHIFT);
1187                         if (hw->mac.type == fm10k_mac_pf)
1188                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1189                                         FM10K_ITR_MASK_SET);
1190                         else
1191                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1192                                         FM10K_ITR_MASK_SET);
1193                 }
1194         }
1195         /* Clean datapath event and queue/vec mapping */
1196         rte_intr_efd_disable(intr_handle);
1197         rte_free(intr_handle->intr_vec);
1198         intr_handle->intr_vec = NULL;
1199 }
1200
1201 static void
1202 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1203 {
1204         int i;
1205
1206         PMD_INIT_FUNC_TRACE();
1207
1208         if (dev->data->tx_queues) {
1209                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1210                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1211
1212                         tx_queue_free(txq);
1213                 }
1214         }
1215
1216         if (dev->data->rx_queues) {
1217                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1218                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1219         }
1220 }
1221
1222 static void
1223 fm10k_dev_close(struct rte_eth_dev *dev)
1224 {
1225         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1226
1227         PMD_INIT_FUNC_TRACE();
1228
1229         fm10k_mbx_lock(hw);
1230         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1231                 MAX_LPORT_NUM, false);
1232         fm10k_mbx_unlock(hw);
1233
1234         /* allow 100ms for device to quiesce */
1235         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1236
1237         /* Stop mailbox service first */
1238         fm10k_close_mbx_service(hw);
1239         fm10k_dev_stop(dev);
1240         fm10k_dev_queue_release(dev);
1241         fm10k_stop_hw(hw);
1242 }
1243
1244 static int
1245 fm10k_link_update(struct rte_eth_dev *dev,
1246         __rte_unused int wait_to_complete)
1247 {
1248         struct fm10k_dev_info *dev_info =
1249                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1250         PMD_INIT_FUNC_TRACE();
1251
1252         /* The speed is ~50Gbps per Gen3 x8 PCIe interface. For now, we
1253          * leave the speed undefined since there is no 50Gbps Ethernet.
1254          */
1255         dev->data->dev_link.link_speed  = 0;
1256         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1257         dev->data->dev_link.link_status =
1258                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1259
1260         return 0;
1261 }
1262
1263 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1264         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1265 {
1266         unsigned i, q;
1267         unsigned count = 0;
1268
1269         if (xstats_names != NULL) {
1270                 /* Note: limit checked in rte_eth_xstats_names() */
1271
1272                 /* Global stats */
1273                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1274                         snprintf(xstats_names[count].name,
1275                                 sizeof(xstats_names[count].name),
1276                                 "%s", fm10k_hw_stats_strings[count].name);
1277                         count++;
1278                 }
1279
1280                 /* PF queue stats */
1281                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1282                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1283                                 snprintf(xstats_names[count].name,
1284                                         sizeof(xstats_names[count].name),
1285                                         "rx_q%u_%s", q,
1286                                         fm10k_hw_stats_rx_q_strings[i].name);
1287                                 count++;
1288                         }
1289                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1290                                 snprintf(xstats_names[count].name,
1291                                         sizeof(xstats_names[count].name),
1292                                         "tx_q%u_%s", q,
1293                                         fm10k_hw_stats_tx_q_strings[i].name);
1294                                 count++;
1295                         }
1296                 }
1297         }
1298         return FM10K_NB_XSTATS;
1299 }
1300
1301 static int
1302 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1303                  unsigned n)
1304 {
1305         struct fm10k_hw_stats *hw_stats =
1306                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1307         unsigned i, q, count = 0;
1308
1309         if (n < FM10K_NB_XSTATS)
1310                 return FM10K_NB_XSTATS;
1311
1312         /* Global stats */
1313         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1314                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1315                         fm10k_hw_stats_strings[count].offset);
1316                 xstats[count].id = count;
1317                 count++;
1318         }
1319
1320         /* PF queue stats */
1321         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1322                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1323                         xstats[count].value =
1324                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1325                                 fm10k_hw_stats_rx_q_strings[i].offset);
1326                         xstats[count].id = count;
1327                         count++;
1328                 }
1329                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1330                         xstats[count].value =
1331                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1332                                 fm10k_hw_stats_tx_q_strings[i].offset);
1333                         xstats[count].id = count;
1334                         count++;
1335                 }
1336         }
1337
1338         return FM10K_NB_XSTATS;
1339 }
1340
1341 static int
1342 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1343 {
1344         uint64_t ipackets, opackets, ibytes, obytes;
1345         struct fm10k_hw *hw =
1346                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1347         struct fm10k_hw_stats *hw_stats =
1348                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1349         int i;
1350
1351         PMD_INIT_FUNC_TRACE();
1352
1353         fm10k_update_hw_stats(hw, hw_stats);
1354
1355         ipackets = opackets = ibytes = obytes = 0;
1356         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1357                 (i < hw->mac.max_queues); ++i) {
1358                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1359                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1360                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1361                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1362                 ipackets += stats->q_ipackets[i];
1363                 opackets += stats->q_opackets[i];
1364                 ibytes   += stats->q_ibytes[i];
1365                 obytes   += stats->q_obytes[i];
1366         }
1367         stats->ipackets = ipackets;
1368         stats->opackets = opackets;
1369         stats->ibytes = ibytes;
1370         stats->obytes = obytes;
1371         return 0;
1372 }
1373
1374 static void
1375 fm10k_stats_reset(struct rte_eth_dev *dev)
1376 {
1377         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1378         struct fm10k_hw_stats *hw_stats =
1379                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1380
1381         PMD_INIT_FUNC_TRACE();
1382
1383         memset(hw_stats, 0, sizeof(*hw_stats));
1384         fm10k_rebind_hw_stats(hw, hw_stats);
1385 }
1386
1387 static void
1388 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1389         struct rte_eth_dev_info *dev_info)
1390 {
1391         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1392         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1393
1394         PMD_INIT_FUNC_TRACE();
1395
1396         dev_info->pci_dev            = pdev;
1397         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1398         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1399         dev_info->max_rx_queues      = hw->mac.max_queues;
1400         dev_info->max_tx_queues      = hw->mac.max_queues;
1401         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1402         dev_info->max_hash_mac_addrs = 0;
1403         dev_info->max_vfs            = pdev->max_vfs;
1404         dev_info->vmdq_pool_base     = 0;
1405         dev_info->vmdq_queue_base    = 0;
1406         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1407         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1408         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1409         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1410                                     dev_info->rx_queue_offload_capa;
1411         dev_info->tx_offload_capa =
1412                 DEV_TX_OFFLOAD_VLAN_INSERT |
1413                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1414                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1415                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1416                 DEV_TX_OFFLOAD_TCP_TSO;
1417
1418         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1419         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1420
1421         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1422                 .rx_thresh = {
1423                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1424                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1425                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1426                 },
1427                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1428                 .rx_drop_en = 0,
1429                 .offloads = 0,
1430         };
1431
1432         dev_info->default_txconf = (struct rte_eth_txconf) {
1433                 .tx_thresh = {
1434                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1435                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1436                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1437                 },
1438                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1439                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1440                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1441         };
1442
1443         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1444                 .nb_max = FM10K_MAX_RX_DESC,
1445                 .nb_min = FM10K_MIN_RX_DESC,
1446                 .nb_align = FM10K_MULT_RX_DESC,
1447         };
1448
1449         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1450                 .nb_max = FM10K_MAX_TX_DESC,
1451                 .nb_min = FM10K_MIN_TX_DESC,
1452                 .nb_align = FM10K_MULT_TX_DESC,
1453                 .nb_seg_max = FM10K_TX_MAX_SEG,
1454                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1455         };
1456
1457         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1458                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1459                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1460 }
1461
1462 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1463 static const uint32_t *
1464 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1465 {
1466         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1467             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1468                 static uint32_t ptypes[] = {
1469                         /* refers to rx_desc_to_ol_flags() */
1470                         RTE_PTYPE_L2_ETHER,
1471                         RTE_PTYPE_L3_IPV4,
1472                         RTE_PTYPE_L3_IPV4_EXT,
1473                         RTE_PTYPE_L3_IPV6,
1474                         RTE_PTYPE_L3_IPV6_EXT,
1475                         RTE_PTYPE_L4_TCP,
1476                         RTE_PTYPE_L4_UDP,
1477                         RTE_PTYPE_UNKNOWN
1478                 };
1479
1480                 return ptypes;
1481         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1482                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1483                 static uint32_t ptypes_vec[] = {
1484                         /* refers to fm10k_desc_to_pktype_v() */
1485                         RTE_PTYPE_L3_IPV4,
1486                         RTE_PTYPE_L3_IPV4_EXT,
1487                         RTE_PTYPE_L3_IPV6,
1488                         RTE_PTYPE_L3_IPV6_EXT,
1489                         RTE_PTYPE_L4_TCP,
1490                         RTE_PTYPE_L4_UDP,
1491                         RTE_PTYPE_TUNNEL_GENEVE,
1492                         RTE_PTYPE_TUNNEL_NVGRE,
1493                         RTE_PTYPE_TUNNEL_VXLAN,
1494                         RTE_PTYPE_TUNNEL_GRE,
1495                         RTE_PTYPE_UNKNOWN
1496                 };
1497
1498                 return ptypes_vec;
1499         }
1500
1501         return NULL;
1502 }
1503 #else
1504 static const uint32_t *
1505 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1506 {
1507         return NULL;
1508 }
1509 #endif
1510
1511 static int
1512 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1513 {
1514         s32 result;
1515         uint16_t mac_num = 0;
1516         uint32_t vid_idx, vid_bit, mac_index;
1517         struct fm10k_hw *hw;
1518         struct fm10k_macvlan_filter_info *macvlan;
1519         struct rte_eth_dev_data *data = dev->data;
1520
1521         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1522         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1523
1524         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1525                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1526                 return -EINVAL;
1527         }
1528
1529         if (vlan_id > ETH_VLAN_ID_MAX) {
1530                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1531                 return -EINVAL;
1532         }
1533
1534         vid_idx = FM10K_VFTA_IDX(vlan_id);
1535         vid_bit = FM10K_VFTA_BIT(vlan_id);
1536         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1537         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1538                 return 0;
1539         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1540         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1541                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1542                         "in the VLAN filter table");
1543                 return -EINVAL;
1544         }
1545
1546         fm10k_mbx_lock(hw);
1547         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1548         fm10k_mbx_unlock(hw);
1549         if (result != FM10K_SUCCESS) {
1550                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1551                 return -EIO;
1552         }
1553
1554         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1555                         (result == FM10K_SUCCESS); mac_index++) {
1556                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1557                         continue;
1558                 if (mac_num > macvlan->mac_num - 1) {
1559                         PMD_INIT_LOG(ERR, "MAC address number "
1560                                         "not match");
1561                         break;
1562                 }
1563                 fm10k_mbx_lock(hw);
1564                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1565                         data->mac_addrs[mac_index].addr_bytes,
1566                         vlan_id, on, 0);
1567                 fm10k_mbx_unlock(hw);
1568                 mac_num++;
1569         }
1570         if (result != FM10K_SUCCESS) {
1571                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1572                 return -EIO;
1573         }
1574
1575         if (on) {
1576                 macvlan->vlan_num++;
1577                 macvlan->vfta[vid_idx] |= vid_bit;
1578         } else {
1579                 macvlan->vlan_num--;
1580                 macvlan->vfta[vid_idx] &= ~vid_bit;
1581         }
1582         return 0;
1583 }
1584
1585 static int
1586 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1587 {
1588         if (mask & ETH_VLAN_STRIP_MASK) {
1589                 if (!(dev->data->dev_conf.rxmode.offloads &
1590                         DEV_RX_OFFLOAD_VLAN_STRIP))
1591                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1592                                         "always on in fm10k");
1593         }
1594
1595         if (mask & ETH_VLAN_EXTEND_MASK) {
1596                 if (dev->data->dev_conf.rxmode.offloads &
1597                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1598                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1599                                         "supported in fm10k");
1600         }
1601
1602         if (mask & ETH_VLAN_FILTER_MASK) {
1603                 if (!(dev->data->dev_conf.rxmode.offloads &
1604                         DEV_RX_OFFLOAD_VLAN_FILTER))
1605                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1606         }
1607
1608         return 0;
1609 }
1610
1611 /* Add/Remove a MAC address, and update filters to main VSI */
1612 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1613                 const u8 *mac, bool add, uint32_t pool)
1614 {
1615         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1616         struct fm10k_macvlan_filter_info *macvlan;
1617         uint32_t i, j, k;
1618
1619         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1620
1621         if (pool != MAIN_VSI_POOL_NUMBER) {
1622                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1623                         "mac to pool %u", pool);
1624                 return;
1625         }
1626         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1627                 if (!macvlan->vfta[j])
1628                         continue;
1629                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1630                         if (!(macvlan->vfta[j] & (1 << k)))
1631                                 continue;
1632                         if (i + 1 > macvlan->vlan_num) {
1633                                 PMD_INIT_LOG(ERR, "vlan number not match");
1634                                 return;
1635                         }
1636                         fm10k_mbx_lock(hw);
1637                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1638                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1639                         fm10k_mbx_unlock(hw);
1640                         i++;
1641                 }
1642         }
1643 }
1644
1645 /* Add/Remove a MAC address, and update filters to VMDQ */
1646 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1647                 const u8 *mac, bool add, uint32_t pool)
1648 {
1649         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1650         struct fm10k_macvlan_filter_info *macvlan;
1651         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1652         uint32_t i;
1653
1654         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1655         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1656
1657         if (pool > macvlan->nb_queue_pools) {
1658                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1659                         " Max pool is %u",
1660                         pool, macvlan->nb_queue_pools);
1661                 return;
1662         }
1663         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1664                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1665                         continue;
1666                 fm10k_mbx_lock(hw);
1667                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1668                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1669                 fm10k_mbx_unlock(hw);
1670         }
1671 }
1672
1673 /* Add/Remove a MAC address, and update filters */
1674 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1675                 const u8 *mac, bool add, uint32_t pool)
1676 {
1677         struct fm10k_macvlan_filter_info *macvlan;
1678
1679         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1680
1681         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1682                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1683         else
1684                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1685
1686         if (add)
1687                 macvlan->mac_num++;
1688         else
1689                 macvlan->mac_num--;
1690 }
1691
1692 /* Add a MAC address, and update filters */
1693 static int
1694 fm10k_macaddr_add(struct rte_eth_dev *dev,
1695                 struct ether_addr *mac_addr,
1696                 uint32_t index,
1697                 uint32_t pool)
1698 {
1699         struct fm10k_macvlan_filter_info *macvlan;
1700
1701         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1702         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1703         macvlan->mac_vmdq_id[index] = pool;
1704         return 0;
1705 }
1706
1707 /* Remove a MAC address, and update filters */
1708 static void
1709 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1710 {
1711         struct rte_eth_dev_data *data = dev->data;
1712         struct fm10k_macvlan_filter_info *macvlan;
1713
1714         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1715         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1716                         FALSE, macvlan->mac_vmdq_id[index]);
1717         macvlan->mac_vmdq_id[index] = 0;
1718 }
1719
1720 static inline int
1721 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1722 {
1723         if ((request < min) || (request > max) || ((request % mult) != 0))
1724                 return -1;
1725         else
1726                 return 0;
1727 }
1728
1729
1730 static inline int
1731 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1732 {
1733         if ((request < min) || (request > max) || ((div % request) != 0))
1734                 return -1;
1735         else
1736                 return 0;
1737 }
1738
1739 static inline int
1740 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1741 {
1742         uint16_t rx_free_thresh;
1743
1744         if (conf->rx_free_thresh == 0)
1745                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1746         else
1747                 rx_free_thresh = conf->rx_free_thresh;
1748
1749         /* make sure the requested threshold satisfies the constraints */
1750         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1751                         FM10K_RX_FREE_THRESH_MAX(q),
1752                         FM10K_RX_FREE_THRESH_DIV(q),
1753                         rx_free_thresh)) {
1754                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1755                         "less than or equal to %u, "
1756                         "greater than or equal to %u, "
1757                         "and a divisor of %u",
1758                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1759                         FM10K_RX_FREE_THRESH_MIN(q),
1760                         FM10K_RX_FREE_THRESH_DIV(q));
1761                 return -EINVAL;
1762         }
1763
1764         q->alloc_thresh = rx_free_thresh;
1765         q->drop_en = conf->rx_drop_en;
1766         q->rx_deferred_start = conf->rx_deferred_start;
1767
1768         return 0;
1769 }
1770
1771 /*
1772  * Hardware requires specific alignment for Rx packet buffers. At
1773  * least one of the following two conditions must be satisfied.
1774  *  1. Address is 512B aligned
1775  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1776  *
1777  * As such, the driver may need to adjust the DMA address within the
1778  * buffer by up to 512B.
1779  *
1780  * return 1 if the element size is valid, otherwise return 0.
1781  */
1782 static int
1783 mempool_element_size_valid(struct rte_mempool *mp)
1784 {
1785         uint32_t min_size;
1786
1787         /* elt_size includes mbuf header and headroom */
1788         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1789                         RTE_PKTMBUF_HEADROOM;
1790
1791         /* account for up to 512B of alignment */
1792         min_size -= FM10K_RX_DATABUF_ALIGN;
1793
1794         /* sanity check for overflow */
1795         if (min_size > mp->elt_size)
1796                 return 0;
1797
1798         /* size is valid */
1799         return 1;
1800 }
1801
1802 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1803 {
1804         RTE_SET_USED(dev);
1805
1806         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1807 }
1808
1809 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1810 {
1811         RTE_SET_USED(dev);
1812
1813         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1814                            DEV_RX_OFFLOAD_VLAN_FILTER |
1815                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1816                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1817                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1818                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1819                            DEV_RX_OFFLOAD_CRC_STRIP   |
1820                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1821 }
1822
1823 static int
1824 fm10k_check_rx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
1825 {
1826         uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
1827         uint64_t queue_supported = fm10k_get_rx_queue_offloads_capa(dev);
1828         uint64_t port_supported = fm10k_get_rx_port_offloads_capa(dev);
1829
1830         if ((requested & (queue_supported | port_supported)) != requested)
1831                 return 0;
1832
1833         if ((port_offloads ^ requested) & port_supported)
1834                 return 0;
1835
1836         return 1;
1837 }
1838
1839 static int
1840 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1841         uint16_t nb_desc, unsigned int socket_id,
1842         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1843 {
1844         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1845         struct fm10k_dev_info *dev_info =
1846                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1847         struct fm10k_rx_queue *q;
1848         const struct rte_memzone *mz;
1849
1850         PMD_INIT_FUNC_TRACE();
1851
1852         if (!fm10k_check_rx_queue_offloads(dev, conf->offloads)) {
1853                 PMD_INIT_LOG(ERR, "%p: Rx queue offloads 0x%" PRIx64
1854                         " don't match port offloads 0x%" PRIx64
1855                         " or supported port offloads 0x%" PRIx64
1856                         " or supported queue offloads 0x%" PRIx64,
1857                         (void *)dev, conf->offloads,
1858                         dev->data->dev_conf.rxmode.offloads,
1859                         fm10k_get_rx_port_offloads_capa(dev),
1860                         fm10k_get_rx_queue_offloads_capa(dev));
1861                 return -ENOTSUP;
1862         }
1863
1864         /* make sure the mempool element size can account for alignment. */
1865         if (!mempool_element_size_valid(mp)) {
1866                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1867                 return -EINVAL;
1868         }
1869
1870         /* make sure a valid number of descriptors have been requested */
1871         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1872                                 FM10K_MULT_RX_DESC, nb_desc)) {
1873                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1874                         "less than or equal to %"PRIu32", "
1875                         "greater than or equal to %u, "
1876                         "and a multiple of %u",
1877                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1878                         FM10K_MULT_RX_DESC);
1879                 return -EINVAL;
1880         }
1881
1882         /*
1883          * if this queue existed already, free the associated memory. The
1884          * queue cannot be reused in case we need to allocate memory on
1885          * different socket than was previously used.
1886          */
1887         if (dev->data->rx_queues[queue_id] != NULL) {
1888                 rx_queue_free(dev->data->rx_queues[queue_id]);
1889                 dev->data->rx_queues[queue_id] = NULL;
1890         }
1891
1892         /* allocate memory for the queue structure */
1893         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1894                                 socket_id);
1895         if (q == NULL) {
1896                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1897                 return -ENOMEM;
1898         }
1899
1900         /* setup queue */
1901         q->mp = mp;
1902         q->nb_desc = nb_desc;
1903         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1904         q->port_id = dev->data->port_id;
1905         q->queue_id = queue_id;
1906         q->tail_ptr = (volatile uint32_t *)
1907                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1908         q->offloads = conf->offloads;
1909         if (handle_rxconf(q, conf))
1910                 return -EINVAL;
1911
1912         /* allocate memory for the software ring */
1913         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1914                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1915                         RTE_CACHE_LINE_SIZE, socket_id);
1916         if (q->sw_ring == NULL) {
1917                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1918                 rte_free(q);
1919                 return -ENOMEM;
1920         }
1921
1922         /*
1923          * allocate memory for the hardware descriptor ring. A memzone large
1924          * enough to hold the maximum ring size is requested to allow for
1925          * resizing in later calls to the queue setup function.
1926          */
1927         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1928                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1929                                       socket_id);
1930         if (mz == NULL) {
1931                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1932                 rte_free(q->sw_ring);
1933                 rte_free(q);
1934                 return -ENOMEM;
1935         }
1936         q->hw_ring = mz->addr;
1937         q->hw_ring_phys_addr = mz->iova;
1938
1939         /* Check if number of descs satisfied Vector requirement */
1940         if (!rte_is_power_of_2(nb_desc)) {
1941                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1942                                     "preconditions - canceling the feature for "
1943                                     "the whole port[%d]",
1944                              q->queue_id, q->port_id);
1945                 dev_info->rx_vec_allowed = false;
1946         } else
1947                 fm10k_rxq_vec_setup(q);
1948
1949         dev->data->rx_queues[queue_id] = q;
1950         return 0;
1951 }
1952
1953 static void
1954 fm10k_rx_queue_release(void *queue)
1955 {
1956         PMD_INIT_FUNC_TRACE();
1957
1958         rx_queue_free(queue);
1959 }
1960
1961 static inline int
1962 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1963 {
1964         uint16_t tx_free_thresh;
1965         uint16_t tx_rs_thresh;
1966
1967         /* constraint MACROs require that tx_free_thresh is configured
1968          * before tx_rs_thresh */
1969         if (conf->tx_free_thresh == 0)
1970                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1971         else
1972                 tx_free_thresh = conf->tx_free_thresh;
1973
1974         /* make sure the requested threshold satisfies the constraints */
1975         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1976                         FM10K_TX_FREE_THRESH_MAX(q),
1977                         FM10K_TX_FREE_THRESH_DIV(q),
1978                         tx_free_thresh)) {
1979                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1980                         "less than or equal to %u, "
1981                         "greater than or equal to %u, "
1982                         "and a divisor of %u",
1983                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1984                         FM10K_TX_FREE_THRESH_MIN(q),
1985                         FM10K_TX_FREE_THRESH_DIV(q));
1986                 return -EINVAL;
1987         }
1988
1989         q->free_thresh = tx_free_thresh;
1990
1991         if (conf->tx_rs_thresh == 0)
1992                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1993         else
1994                 tx_rs_thresh = conf->tx_rs_thresh;
1995
1996         q->tx_deferred_start = conf->tx_deferred_start;
1997
1998         /* make sure the requested threshold satisfies the constraints */
1999         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
2000                         FM10K_TX_RS_THRESH_MAX(q),
2001                         FM10K_TX_RS_THRESH_DIV(q),
2002                         tx_rs_thresh)) {
2003                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
2004                         "less than or equal to %u, "
2005                         "greater than or equal to %u, "
2006                         "and a divisor of %u",
2007                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
2008                         FM10K_TX_RS_THRESH_MIN(q),
2009                         FM10K_TX_RS_THRESH_DIV(q));
2010                 return -EINVAL;
2011         }
2012
2013         q->rs_thresh = tx_rs_thresh;
2014
2015         return 0;
2016 }
2017
2018 static int
2019 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2020         uint16_t nb_desc, unsigned int socket_id,
2021         const struct rte_eth_txconf *conf)
2022 {
2023         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2024         struct fm10k_tx_queue *q;
2025         const struct rte_memzone *mz;
2026
2027         PMD_INIT_FUNC_TRACE();
2028
2029         /* make sure a valid number of descriptors have been requested */
2030         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2031                                 FM10K_MULT_TX_DESC, nb_desc)) {
2032                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2033                         "less than or equal to %"PRIu32", "
2034                         "greater than or equal to %u, "
2035                         "and a multiple of %u",
2036                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2037                         FM10K_MULT_TX_DESC);
2038                 return -EINVAL;
2039         }
2040
2041         /*
2042          * if this queue existed already, free the associated memory. The
2043          * queue cannot be reused in case we need to allocate memory on
2044          * different socket than was previously used.
2045          */
2046         if (dev->data->tx_queues[queue_id] != NULL) {
2047                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2048
2049                 tx_queue_free(txq);
2050                 dev->data->tx_queues[queue_id] = NULL;
2051         }
2052
2053         /* allocate memory for the queue structure */
2054         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2055                                 socket_id);
2056         if (q == NULL) {
2057                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2058                 return -ENOMEM;
2059         }
2060
2061         /* setup queue */
2062         q->nb_desc = nb_desc;
2063         q->port_id = dev->data->port_id;
2064         q->queue_id = queue_id;
2065         q->txq_flags = conf->txq_flags;
2066         q->ops = &def_txq_ops;
2067         q->tail_ptr = (volatile uint32_t *)
2068                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2069         if (handle_txconf(q, conf))
2070                 return -EINVAL;
2071
2072         /* allocate memory for the software ring */
2073         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2074                                         nb_desc * sizeof(struct rte_mbuf *),
2075                                         RTE_CACHE_LINE_SIZE, socket_id);
2076         if (q->sw_ring == NULL) {
2077                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2078                 rte_free(q);
2079                 return -ENOMEM;
2080         }
2081
2082         /*
2083          * allocate memory for the hardware descriptor ring. A memzone large
2084          * enough to hold the maximum ring size is requested to allow for
2085          * resizing in later calls to the queue setup function.
2086          */
2087         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2088                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2089                                       socket_id);
2090         if (mz == NULL) {
2091                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2092                 rte_free(q->sw_ring);
2093                 rte_free(q);
2094                 return -ENOMEM;
2095         }
2096         q->hw_ring = mz->addr;
2097         q->hw_ring_phys_addr = mz->iova;
2098
2099         /*
2100          * allocate memory for the RS bit tracker. Enough slots to hold the
2101          * descriptor index for each RS bit needing to be set are required.
2102          */
2103         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2104                                 ((nb_desc + 1) / q->rs_thresh) *
2105                                 sizeof(uint16_t),
2106                                 RTE_CACHE_LINE_SIZE, socket_id);
2107         if (q->rs_tracker.list == NULL) {
2108                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2109                 rte_free(q->sw_ring);
2110                 rte_free(q);
2111                 return -ENOMEM;
2112         }
2113
2114         dev->data->tx_queues[queue_id] = q;
2115         return 0;
2116 }
2117
2118 static void
2119 fm10k_tx_queue_release(void *queue)
2120 {
2121         struct fm10k_tx_queue *q = queue;
2122         PMD_INIT_FUNC_TRACE();
2123
2124         tx_queue_free(q);
2125 }
2126
2127 static int
2128 fm10k_reta_update(struct rte_eth_dev *dev,
2129                         struct rte_eth_rss_reta_entry64 *reta_conf,
2130                         uint16_t reta_size)
2131 {
2132         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2133         uint16_t i, j, idx, shift;
2134         uint8_t mask;
2135         uint32_t reta;
2136
2137         PMD_INIT_FUNC_TRACE();
2138
2139         if (reta_size > FM10K_MAX_RSS_INDICES) {
2140                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2141                         "(%d) doesn't match the number hardware can supported "
2142                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2143                 return -EINVAL;
2144         }
2145
2146         /*
2147          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2148          * 128-entries in 32 registers
2149          */
2150         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2151                 idx = i / RTE_RETA_GROUP_SIZE;
2152                 shift = i % RTE_RETA_GROUP_SIZE;
2153                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2154                                 BIT_MASK_PER_UINT32);
2155                 if (mask == 0)
2156                         continue;
2157
2158                 reta = 0;
2159                 if (mask != BIT_MASK_PER_UINT32)
2160                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2161
2162                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2163                         if (mask & (0x1 << j)) {
2164                                 if (mask != 0xF)
2165                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2166                                 reta |= reta_conf[idx].reta[shift + j] <<
2167                                                 (CHAR_BIT * j);
2168                         }
2169                 }
2170                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2171         }
2172
2173         return 0;
2174 }
2175
2176 static int
2177 fm10k_reta_query(struct rte_eth_dev *dev,
2178                         struct rte_eth_rss_reta_entry64 *reta_conf,
2179                         uint16_t reta_size)
2180 {
2181         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2182         uint16_t i, j, idx, shift;
2183         uint8_t mask;
2184         uint32_t reta;
2185
2186         PMD_INIT_FUNC_TRACE();
2187
2188         if (reta_size < FM10K_MAX_RSS_INDICES) {
2189                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2190                         "(%d) doesn't match the number hardware can supported "
2191                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2192                 return -EINVAL;
2193         }
2194
2195         /*
2196          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2197          * 128-entries in 32 registers
2198          */
2199         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2200                 idx = i / RTE_RETA_GROUP_SIZE;
2201                 shift = i % RTE_RETA_GROUP_SIZE;
2202                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2203                                 BIT_MASK_PER_UINT32);
2204                 if (mask == 0)
2205                         continue;
2206
2207                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2208                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2209                         if (mask & (0x1 << j))
2210                                 reta_conf[idx].reta[shift + j] = ((reta >>
2211                                         CHAR_BIT * j) & UINT8_MAX);
2212                 }
2213         }
2214
2215         return 0;
2216 }
2217
2218 static int
2219 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2220         struct rte_eth_rss_conf *rss_conf)
2221 {
2222         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2223         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2224         uint32_t mrqc;
2225         uint64_t hf = rss_conf->rss_hf;
2226         int i;
2227
2228         PMD_INIT_FUNC_TRACE();
2229
2230         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2231                                 FM10K_RSSRK_ENTRIES_PER_REG))
2232                 return -EINVAL;
2233
2234         if (hf == 0)
2235                 return -EINVAL;
2236
2237         mrqc = 0;
2238         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2239         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2240         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2241         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2242         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2243         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2244         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2245         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2246         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2247
2248         /* If the mapping doesn't fit any supported, return */
2249         if (mrqc == 0)
2250                 return -EINVAL;
2251
2252         if (key != NULL)
2253                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2254                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2255
2256         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2257
2258         return 0;
2259 }
2260
2261 static int
2262 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2263         struct rte_eth_rss_conf *rss_conf)
2264 {
2265         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2266         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2267         uint32_t mrqc;
2268         uint64_t hf;
2269         int i;
2270
2271         PMD_INIT_FUNC_TRACE();
2272
2273         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2274                                 FM10K_RSSRK_ENTRIES_PER_REG))
2275                 return -EINVAL;
2276
2277         if (key != NULL)
2278                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2279                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2280
2281         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2282         hf = 0;
2283         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2284         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2285         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2286         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2287         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2288         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2289         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2290         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2291         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2292
2293         rss_conf->rss_hf = hf;
2294
2295         return 0;
2296 }
2297
2298 static void
2299 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2300 {
2301         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2302         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2303
2304         /* Bind all local non-queue interrupt to vector 0 */
2305         int_map |= FM10K_MISC_VEC_ID;
2306
2307         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2308         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2309         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2310         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2311         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2312         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2313
2314         /* Enable misc causes */
2315         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2316                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2317                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2318                                 FM10K_EIMR_ENABLE(MAILBOX) |
2319                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2320                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2321                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2322                                 FM10K_EIMR_ENABLE(VFLR));
2323
2324         /* Enable ITR 0 */
2325         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2326                                         FM10K_ITR_MASK_CLEAR);
2327         FM10K_WRITE_FLUSH(hw);
2328 }
2329
2330 static void
2331 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2332 {
2333         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2334         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2335
2336         int_map |= FM10K_MISC_VEC_ID;
2337
2338         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2339         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2340         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2341         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2342         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2343         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2344
2345         /* Disable misc causes */
2346         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2347                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2348                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2349                                 FM10K_EIMR_DISABLE(MAILBOX) |
2350                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2351                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2352                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2353                                 FM10K_EIMR_DISABLE(VFLR));
2354
2355         /* Disable ITR 0 */
2356         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2357         FM10K_WRITE_FLUSH(hw);
2358 }
2359
2360 static void
2361 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2362 {
2363         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2364         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2365
2366         /* Bind all local non-queue interrupt to vector 0 */
2367         int_map |= FM10K_MISC_VEC_ID;
2368
2369         /* Only INT 0 available, other 15 are reserved. */
2370         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2371
2372         /* Enable ITR 0 */
2373         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2374                                         FM10K_ITR_MASK_CLEAR);
2375         FM10K_WRITE_FLUSH(hw);
2376 }
2377
2378 static void
2379 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2380 {
2381         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2382         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2383
2384         int_map |= FM10K_MISC_VEC_ID;
2385
2386         /* Only INT 0 available, other 15 are reserved. */
2387         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2388
2389         /* Disable ITR 0 */
2390         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2391         FM10K_WRITE_FLUSH(hw);
2392 }
2393
2394 static int
2395 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2396 {
2397         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2398         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2399
2400         /* Enable ITR */
2401         if (hw->mac.type == fm10k_mac_pf)
2402                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2403                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2404         else
2405                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2406                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2407         rte_intr_enable(&pdev->intr_handle);
2408         return 0;
2409 }
2410
2411 static int
2412 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2413 {
2414         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2415         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2416
2417         /* Disable ITR */
2418         if (hw->mac.type == fm10k_mac_pf)
2419                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2420                         FM10K_ITR_MASK_SET);
2421         else
2422                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2423                         FM10K_ITR_MASK_SET);
2424         return 0;
2425 }
2426
2427 static int
2428 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2429 {
2430         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2431         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2432         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2433         uint32_t intr_vector, vec;
2434         uint16_t queue_id;
2435         int result = 0;
2436
2437         /* fm10k needs one separate interrupt for mailbox,
2438          * so only drivers which support multiple interrupt vectors
2439          * e.g. vfio-pci can work for fm10k interrupt mode
2440          */
2441         if (!rte_intr_cap_multiple(intr_handle) ||
2442                         dev->data->dev_conf.intr_conf.rxq == 0)
2443                 return result;
2444
2445         intr_vector = dev->data->nb_rx_queues;
2446
2447         /* disable interrupt first */
2448         rte_intr_disable(intr_handle);
2449         if (hw->mac.type == fm10k_mac_pf)
2450                 fm10k_dev_disable_intr_pf(dev);
2451         else
2452                 fm10k_dev_disable_intr_vf(dev);
2453
2454         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2455                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2456                 result = -EIO;
2457         }
2458
2459         if (rte_intr_dp_is_en(intr_handle) && !result) {
2460                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2461                         dev->data->nb_rx_queues * sizeof(int), 0);
2462                 if (intr_handle->intr_vec) {
2463                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2464                                         queue_id < dev->data->nb_rx_queues;
2465                                         queue_id++) {
2466                                 intr_handle->intr_vec[queue_id] = vec;
2467                                 if (vec < intr_handle->nb_efd - 1
2468                                                 + FM10K_RX_VEC_START)
2469                                         vec++;
2470                         }
2471                 } else {
2472                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2473                                 " intr_vec", dev->data->nb_rx_queues);
2474                         rte_intr_efd_disable(intr_handle);
2475                         result = -ENOMEM;
2476                 }
2477         }
2478
2479         if (hw->mac.type == fm10k_mac_pf)
2480                 fm10k_dev_enable_intr_pf(dev);
2481         else
2482                 fm10k_dev_enable_intr_vf(dev);
2483         rte_intr_enable(intr_handle);
2484         hw->mac.ops.update_int_moderator(hw);
2485         return result;
2486 }
2487
2488 static int
2489 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2490 {
2491         struct fm10k_fault fault;
2492         int err;
2493         const char *estr = "Unknown error";
2494
2495         /* Process PCA fault */
2496         if (eicr & FM10K_EICR_PCA_FAULT) {
2497                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2498                 if (err)
2499                         goto error;
2500                 switch (fault.type) {
2501                 case PCA_NO_FAULT:
2502                         estr = "PCA_NO_FAULT"; break;
2503                 case PCA_UNMAPPED_ADDR:
2504                         estr = "PCA_UNMAPPED_ADDR"; break;
2505                 case PCA_BAD_QACCESS_PF:
2506                         estr = "PCA_BAD_QACCESS_PF"; break;
2507                 case PCA_BAD_QACCESS_VF:
2508                         estr = "PCA_BAD_QACCESS_VF"; break;
2509                 case PCA_MALICIOUS_REQ:
2510                         estr = "PCA_MALICIOUS_REQ"; break;
2511                 case PCA_POISONED_TLP:
2512                         estr = "PCA_POISONED_TLP"; break;
2513                 case PCA_TLP_ABORT:
2514                         estr = "PCA_TLP_ABORT"; break;
2515                 default:
2516                         goto error;
2517                 }
2518                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2519                         estr, fault.func ? "VF" : "PF", fault.func,
2520                         fault.address, fault.specinfo);
2521         }
2522
2523         /* Process THI fault */
2524         if (eicr & FM10K_EICR_THI_FAULT) {
2525                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2526                 if (err)
2527                         goto error;
2528                 switch (fault.type) {
2529                 case THI_NO_FAULT:
2530                         estr = "THI_NO_FAULT"; break;
2531                 case THI_MAL_DIS_Q_FAULT:
2532                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2533                 default:
2534                         goto error;
2535                 }
2536                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2537                         estr, fault.func ? "VF" : "PF", fault.func,
2538                         fault.address, fault.specinfo);
2539         }
2540
2541         /* Process FUM fault */
2542         if (eicr & FM10K_EICR_FUM_FAULT) {
2543                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2544                 if (err)
2545                         goto error;
2546                 switch (fault.type) {
2547                 case FUM_NO_FAULT:
2548                         estr = "FUM_NO_FAULT"; break;
2549                 case FUM_UNMAPPED_ADDR:
2550                         estr = "FUM_UNMAPPED_ADDR"; break;
2551                 case FUM_POISONED_TLP:
2552                         estr = "FUM_POISONED_TLP"; break;
2553                 case FUM_BAD_VF_QACCESS:
2554                         estr = "FUM_BAD_VF_QACCESS"; break;
2555                 case FUM_ADD_DECODE_ERR:
2556                         estr = "FUM_ADD_DECODE_ERR"; break;
2557                 case FUM_RO_ERROR:
2558                         estr = "FUM_RO_ERROR"; break;
2559                 case FUM_QPRC_CRC_ERROR:
2560                         estr = "FUM_QPRC_CRC_ERROR"; break;
2561                 case FUM_CSR_TIMEOUT:
2562                         estr = "FUM_CSR_TIMEOUT"; break;
2563                 case FUM_INVALID_TYPE:
2564                         estr = "FUM_INVALID_TYPE"; break;
2565                 case FUM_INVALID_LENGTH:
2566                         estr = "FUM_INVALID_LENGTH"; break;
2567                 case FUM_INVALID_BE:
2568                         estr = "FUM_INVALID_BE"; break;
2569                 case FUM_INVALID_ALIGN:
2570                         estr = "FUM_INVALID_ALIGN"; break;
2571                 default:
2572                         goto error;
2573                 }
2574                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2575                         estr, fault.func ? "VF" : "PF", fault.func,
2576                         fault.address, fault.specinfo);
2577         }
2578
2579         return 0;
2580 error:
2581         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2582         return err;
2583 }
2584
2585 /**
2586  * PF interrupt handler triggered by NIC for handling specific interrupt.
2587  *
2588  * @param handle
2589  *  Pointer to interrupt handle.
2590  * @param param
2591  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2592  *
2593  * @return
2594  *  void
2595  */
2596 static void
2597 fm10k_dev_interrupt_handler_pf(void *param)
2598 {
2599         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2600         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2601         uint32_t cause, status;
2602         struct fm10k_dev_info *dev_info =
2603                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2604         int status_mbx;
2605         s32 err;
2606
2607         if (hw->mac.type != fm10k_mac_pf)
2608                 return;
2609
2610         cause = FM10K_READ_REG(hw, FM10K_EICR);
2611
2612         /* Handle PCI fault cases */
2613         if (cause & FM10K_EICR_FAULT_MASK) {
2614                 PMD_INIT_LOG(ERR, "INT: find fault!");
2615                 fm10k_dev_handle_fault(hw, cause);
2616         }
2617
2618         /* Handle switch up/down */
2619         if (cause & FM10K_EICR_SWITCHNOTREADY)
2620                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2621
2622         if (cause & FM10K_EICR_SWITCHREADY) {
2623                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2624                 if (dev_info->sm_down == 1) {
2625                         fm10k_mbx_lock(hw);
2626
2627                         /* For recreating logical ports */
2628                         status_mbx = hw->mac.ops.update_lport_state(hw,
2629                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2630                         if (status_mbx == FM10K_SUCCESS)
2631                                 PMD_INIT_LOG(INFO,
2632                                         "INT: Recreated Logical port");
2633                         else
2634                                 PMD_INIT_LOG(INFO,
2635                                         "INT: Logical ports weren't recreated");
2636
2637                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2638                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2639                         if (status_mbx != FM10K_SUCCESS)
2640                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2641
2642                         fm10k_mbx_unlock(hw);
2643
2644                         /* first clear the internal SW recording structure */
2645                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2646                                                 ETH_MQ_RX_VMDQ_FLAG))
2647                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2648                                         false);
2649
2650                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2651                                         MAIN_VSI_POOL_NUMBER);
2652
2653                         /*
2654                          * Add default mac address and vlan for the logical
2655                          * ports that have been created, leave to the
2656                          * application to fully recover Rx filtering.
2657                          */
2658                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2659                                         MAIN_VSI_POOL_NUMBER);
2660
2661                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2662                                                 ETH_MQ_RX_VMDQ_FLAG))
2663                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2664                                         true);
2665
2666                         dev_info->sm_down = 0;
2667                         _rte_eth_dev_callback_process(dev,
2668                                         RTE_ETH_EVENT_INTR_LSC,
2669                                         NULL);
2670                 }
2671         }
2672
2673         /* Handle mailbox message */
2674         fm10k_mbx_lock(hw);
2675         err = hw->mbx.ops.process(hw, &hw->mbx);
2676         fm10k_mbx_unlock(hw);
2677
2678         if (err == FM10K_ERR_RESET_REQUESTED) {
2679                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2680                 dev_info->sm_down = 1;
2681                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2682                                 NULL);
2683         }
2684
2685         /* Handle SRAM error */
2686         if (cause & FM10K_EICR_SRAMERROR) {
2687                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2688
2689                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2690                 /* Write to clear pending bits */
2691                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2692
2693                 /* Todo: print out error message after shared code  updates */
2694         }
2695
2696         /* Clear these 3 events if having any */
2697         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2698                  FM10K_EICR_SWITCHREADY;
2699         if (cause)
2700                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2701
2702         /* Re-enable interrupt from device side */
2703         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2704                                         FM10K_ITR_MASK_CLEAR);
2705         /* Re-enable interrupt from host side */
2706         rte_intr_enable(dev->intr_handle);
2707 }
2708
2709 /**
2710  * VF interrupt handler triggered by NIC for handling specific interrupt.
2711  *
2712  * @param handle
2713  *  Pointer to interrupt handle.
2714  * @param param
2715  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2716  *
2717  * @return
2718  *  void
2719  */
2720 static void
2721 fm10k_dev_interrupt_handler_vf(void *param)
2722 {
2723         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2724         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2725         struct fm10k_mbx_info *mbx = &hw->mbx;
2726         struct fm10k_dev_info *dev_info =
2727                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2728         const enum fm10k_mbx_state state = mbx->state;
2729         int status_mbx;
2730
2731         if (hw->mac.type != fm10k_mac_vf)
2732                 return;
2733
2734         /* Handle mailbox message if lock is acquired */
2735         fm10k_mbx_lock(hw);
2736         hw->mbx.ops.process(hw, &hw->mbx);
2737         fm10k_mbx_unlock(hw);
2738
2739         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2740                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2741
2742                 fm10k_mbx_lock(hw);
2743                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2744                                 MAX_LPORT_NUM, 1);
2745                 fm10k_mbx_unlock(hw);
2746
2747                 /* Setting reset flag */
2748                 dev_info->sm_down = 1;
2749                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2750                                 NULL);
2751         }
2752
2753         if (dev_info->sm_down == 1 &&
2754                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2755                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2756                 fm10k_mbx_lock(hw);
2757                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2758                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2759                 if (status_mbx != FM10K_SUCCESS)
2760                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2761                 fm10k_mbx_unlock(hw);
2762
2763                 /* first clear the internal SW recording structure */
2764                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2765                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2766                                 MAIN_VSI_POOL_NUMBER);
2767
2768                 /*
2769                  * Add default mac address and vlan for the logical ports that
2770                  * have been created, leave to the application to fully recover
2771                  * Rx filtering.
2772                  */
2773                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2774                                 MAIN_VSI_POOL_NUMBER);
2775                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2776
2777                 dev_info->sm_down = 0;
2778                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2779                                 NULL);
2780         }
2781
2782         /* Re-enable interrupt from device side */
2783         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2784                                         FM10K_ITR_MASK_CLEAR);
2785         /* Re-enable interrupt from host side */
2786         rte_intr_enable(dev->intr_handle);
2787 }
2788
2789 /* Mailbox message handler in VF */
2790 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2791         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2792         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2793         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2794         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2795 };
2796
2797 static int
2798 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2799 {
2800         int err = 0;
2801
2802         /* Initialize mailbox lock */
2803         fm10k_mbx_initlock(hw);
2804
2805         /* Replace default message handler with new ones */
2806         if (hw->mac.type == fm10k_mac_vf)
2807                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2808
2809         if (err) {
2810                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2811                                 err);
2812                 return err;
2813         }
2814         /* Connect to SM for PF device or PF for VF device */
2815         return hw->mbx.ops.connect(hw, &hw->mbx);
2816 }
2817
2818 static void
2819 fm10k_close_mbx_service(struct fm10k_hw *hw)
2820 {
2821         /* Disconnect from SM for PF device or PF for VF device */
2822         hw->mbx.ops.disconnect(hw, &hw->mbx);
2823 }
2824
2825 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2826         .dev_configure          = fm10k_dev_configure,
2827         .dev_start              = fm10k_dev_start,
2828         .dev_stop               = fm10k_dev_stop,
2829         .dev_close              = fm10k_dev_close,
2830         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2831         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2832         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2833         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2834         .stats_get              = fm10k_stats_get,
2835         .xstats_get             = fm10k_xstats_get,
2836         .xstats_get_names       = fm10k_xstats_get_names,
2837         .stats_reset            = fm10k_stats_reset,
2838         .xstats_reset           = fm10k_stats_reset,
2839         .link_update            = fm10k_link_update,
2840         .dev_infos_get          = fm10k_dev_infos_get,
2841         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2842         .vlan_filter_set        = fm10k_vlan_filter_set,
2843         .vlan_offload_set       = fm10k_vlan_offload_set,
2844         .mac_addr_add           = fm10k_macaddr_add,
2845         .mac_addr_remove        = fm10k_macaddr_remove,
2846         .rx_queue_start         = fm10k_dev_rx_queue_start,
2847         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2848         .tx_queue_start         = fm10k_dev_tx_queue_start,
2849         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2850         .rx_queue_setup         = fm10k_rx_queue_setup,
2851         .rx_queue_release       = fm10k_rx_queue_release,
2852         .tx_queue_setup         = fm10k_tx_queue_setup,
2853         .tx_queue_release       = fm10k_tx_queue_release,
2854         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2855         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2856         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2857         .reta_update            = fm10k_reta_update,
2858         .reta_query             = fm10k_reta_query,
2859         .rss_hash_update        = fm10k_rss_hash_update,
2860         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2861 };
2862
2863 static int ftag_check_handler(__rte_unused const char *key,
2864                 const char *value, __rte_unused void *opaque)
2865 {
2866         if (strcmp(value, "1"))
2867                 return -1;
2868
2869         return 0;
2870 }
2871
2872 static int
2873 fm10k_check_ftag(struct rte_devargs *devargs)
2874 {
2875         struct rte_kvargs *kvlist;
2876         const char *ftag_key = "enable_ftag";
2877
2878         if (devargs == NULL)
2879                 return 0;
2880
2881         kvlist = rte_kvargs_parse(devargs->args, NULL);
2882         if (kvlist == NULL)
2883                 return 0;
2884
2885         if (!rte_kvargs_count(kvlist, ftag_key)) {
2886                 rte_kvargs_free(kvlist);
2887                 return 0;
2888         }
2889         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2890         if (rte_kvargs_process(kvlist, ftag_key,
2891                                 ftag_check_handler, NULL) < 0) {
2892                 rte_kvargs_free(kvlist);
2893                 return 0;
2894         }
2895         rte_kvargs_free(kvlist);
2896
2897         return 1;
2898 }
2899
2900 static uint16_t
2901 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2902                     uint16_t nb_pkts)
2903 {
2904         uint16_t nb_tx = 0;
2905         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2906
2907         while (nb_pkts) {
2908                 uint16_t ret, num;
2909
2910                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2911                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2912                                                  num);
2913                 nb_tx += ret;
2914                 nb_pkts -= ret;
2915                 if (ret < num)
2916                         break;
2917         }
2918
2919         return nb_tx;
2920 }
2921
2922 static void __attribute__((cold))
2923 fm10k_set_tx_function(struct rte_eth_dev *dev)
2924 {
2925         struct fm10k_tx_queue *txq;
2926         int i;
2927         int use_sse = 1;
2928         uint16_t tx_ftag_en = 0;
2929
2930         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2931                 /* primary process has set the ftag flag and txq_flags */
2932                 txq = dev->data->tx_queues[0];
2933                 if (fm10k_tx_vec_condition_check(txq)) {
2934                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2935                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2936                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2937                 } else {
2938                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2939                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2940                         dev->tx_pkt_prepare = NULL;
2941                 }
2942                 return;
2943         }
2944
2945         if (fm10k_check_ftag(dev->device->devargs))
2946                 tx_ftag_en = 1;
2947
2948         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2949                 txq = dev->data->tx_queues[i];
2950                 txq->tx_ftag_en = tx_ftag_en;
2951                 /* Check if Vector Tx is satisfied */
2952                 if (fm10k_tx_vec_condition_check(txq))
2953                         use_sse = 0;
2954         }
2955
2956         if (use_sse) {
2957                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2958                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2959                         txq = dev->data->tx_queues[i];
2960                         fm10k_txq_vec_setup(txq);
2961                 }
2962                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2963                 dev->tx_pkt_prepare = NULL;
2964         } else {
2965                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2966                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2967                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2968         }
2969 }
2970
2971 static void __attribute__((cold))
2972 fm10k_set_rx_function(struct rte_eth_dev *dev)
2973 {
2974         struct fm10k_dev_info *dev_info =
2975                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2976         uint16_t i, rx_using_sse;
2977         uint16_t rx_ftag_en = 0;
2978
2979         if (fm10k_check_ftag(dev->device->devargs))
2980                 rx_ftag_en = 1;
2981
2982         /* In order to allow Vector Rx there are a few configuration
2983          * conditions to be met.
2984          */
2985         if (!fm10k_rx_vec_condition_check(dev) &&
2986                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2987                 if (dev->data->scattered_rx)
2988                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2989                 else
2990                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2991         } else if (dev->data->scattered_rx)
2992                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2993         else
2994                 dev->rx_pkt_burst = fm10k_recv_pkts;
2995
2996         rx_using_sse =
2997                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2998                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2999
3000         if (rx_using_sse)
3001                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3002         else
3003                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3004
3005         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3006                 return;
3007
3008         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3009                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3010
3011                 rxq->rx_using_sse = rx_using_sse;
3012                 rxq->rx_ftag_en = rx_ftag_en;
3013         }
3014 }
3015
3016 static void
3017 fm10k_params_init(struct rte_eth_dev *dev)
3018 {
3019         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3020         struct fm10k_dev_info *info =
3021                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3022
3023         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3024          * there is no way to get link status without reading BAR4.  Until this
3025          * works, assume we have maximum bandwidth.
3026          * @todo - fix bus info
3027          */
3028         hw->bus_caps.speed = fm10k_bus_speed_8000;
3029         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3030         hw->bus_caps.payload = fm10k_bus_payload_512;
3031         hw->bus.speed = fm10k_bus_speed_8000;
3032         hw->bus.width = fm10k_bus_width_pcie_x8;
3033         hw->bus.payload = fm10k_bus_payload_256;
3034
3035         info->rx_vec_allowed = true;
3036 }
3037
3038 static int
3039 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3040 {
3041         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3042         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3043         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3044         int diag, i;
3045         struct fm10k_macvlan_filter_info *macvlan;
3046
3047         PMD_INIT_FUNC_TRACE();
3048
3049         dev->dev_ops = &fm10k_eth_dev_ops;
3050         dev->rx_pkt_burst = &fm10k_recv_pkts;
3051         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3052         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3053
3054         /*
3055          * Primary process does the whole initialization, for secondary
3056          * processes, we just select the same Rx and Tx function as primary.
3057          */
3058         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3059                 fm10k_set_rx_function(dev);
3060                 fm10k_set_tx_function(dev);
3061                 return 0;
3062         }
3063
3064         rte_eth_copy_pci_info(dev, pdev);
3065
3066         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3067         memset(macvlan, 0, sizeof(*macvlan));
3068         /* Vendor and Device ID need to be set before init of shared code */
3069         memset(hw, 0, sizeof(*hw));
3070         hw->device_id = pdev->id.device_id;
3071         hw->vendor_id = pdev->id.vendor_id;
3072         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3073         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3074         hw->revision_id = 0;
3075         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3076         if (hw->hw_addr == NULL) {
3077                 PMD_INIT_LOG(ERR, "Bad mem resource."
3078                         " Try to blacklist unused devices.");
3079                 return -EIO;
3080         }
3081
3082         /* Store fm10k_adapter pointer */
3083         hw->back = dev->data->dev_private;
3084
3085         /* Initialize the shared code */
3086         diag = fm10k_init_shared_code(hw);
3087         if (diag != FM10K_SUCCESS) {
3088                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3089                 return -EIO;
3090         }
3091
3092         /* Initialize parameters */
3093         fm10k_params_init(dev);
3094
3095         /* Initialize the hw */
3096         diag = fm10k_init_hw(hw);
3097         if (diag != FM10K_SUCCESS) {
3098                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3099                 return -EIO;
3100         }
3101
3102         /* Initialize MAC address(es) */
3103         dev->data->mac_addrs = rte_zmalloc("fm10k",
3104                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3105         if (dev->data->mac_addrs == NULL) {
3106                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3107                 return -ENOMEM;
3108         }
3109
3110         diag = fm10k_read_mac_addr(hw);
3111
3112         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3113                         &dev->data->mac_addrs[0]);
3114
3115         if (diag != FM10K_SUCCESS ||
3116                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3117
3118                 /* Generate a random addr */
3119                 eth_random_addr(hw->mac.addr);
3120                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3121                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3122                 &dev->data->mac_addrs[0]);
3123         }
3124
3125         /* Reset the hw statistics */
3126         fm10k_stats_reset(dev);
3127
3128         /* Reset the hw */
3129         diag = fm10k_reset_hw(hw);
3130         if (diag != FM10K_SUCCESS) {
3131                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3132                 return -EIO;
3133         }
3134
3135         /* Setup mailbox service */
3136         diag = fm10k_setup_mbx_service(hw);
3137         if (diag != FM10K_SUCCESS) {
3138                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3139                 return -EIO;
3140         }
3141
3142         /*PF/VF has different interrupt handling mechanism */
3143         if (hw->mac.type == fm10k_mac_pf) {
3144                 /* register callback func to eal lib */
3145                 rte_intr_callback_register(intr_handle,
3146                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3147
3148                 /* enable MISC interrupt */
3149                 fm10k_dev_enable_intr_pf(dev);
3150         } else { /* VF */
3151                 rte_intr_callback_register(intr_handle,
3152                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3153
3154                 fm10k_dev_enable_intr_vf(dev);
3155         }
3156
3157         /* Enable intr after callback registered */
3158         rte_intr_enable(intr_handle);
3159
3160         hw->mac.ops.update_int_moderator(hw);
3161
3162         /* Make sure Switch Manager is ready before going forward. */
3163         if (hw->mac.type == fm10k_mac_pf) {
3164                 int switch_ready = 0;
3165
3166                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3167                         fm10k_mbx_lock(hw);
3168                         hw->mac.ops.get_host_state(hw, &switch_ready);
3169                         fm10k_mbx_unlock(hw);
3170                         if (switch_ready)
3171                                 break;
3172                         /* Delay some time to acquire async LPORT_MAP info. */
3173                         rte_delay_us(WAIT_SWITCH_MSG_US);
3174                 }
3175
3176                 if (switch_ready == 0) {
3177                         PMD_INIT_LOG(ERR, "switch is not ready");
3178                         return -1;
3179                 }
3180         }
3181
3182         /*
3183          * Below function will trigger operations on mailbox, acquire lock to
3184          * avoid race condition from interrupt handler. Operations on mailbox
3185          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3186          * will handle and generate an interrupt to our side. Then,  FIFO in
3187          * mailbox will be touched.
3188          */
3189         fm10k_mbx_lock(hw);
3190         /* Enable port first */
3191         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3192                                         MAX_LPORT_NUM, 1);
3193
3194         /* Set unicast mode by default. App can change to other mode in other
3195          * API func.
3196          */
3197         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3198                                         FM10K_XCAST_MODE_NONE);
3199
3200         fm10k_mbx_unlock(hw);
3201
3202         /* Make sure default VID is ready before going forward. */
3203         if (hw->mac.type == fm10k_mac_pf) {
3204                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3205                         if (hw->mac.default_vid)
3206                                 break;
3207                         /* Delay some time to acquire async port VLAN info. */
3208                         rte_delay_us(WAIT_SWITCH_MSG_US);
3209                 }
3210
3211                 if (!hw->mac.default_vid) {
3212                         PMD_INIT_LOG(ERR, "default VID is not ready");
3213                         return -1;
3214                 }
3215         }
3216
3217         /* Add default mac address */
3218         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3219                 MAIN_VSI_POOL_NUMBER);
3220
3221         return 0;
3222 }
3223
3224 static int
3225 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3226 {
3227         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3228         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3229         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3230         PMD_INIT_FUNC_TRACE();
3231
3232         /* only uninitialize in the primary process */
3233         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3234                 return 0;
3235
3236         /* safe to close dev here */
3237         fm10k_dev_close(dev);
3238
3239         dev->dev_ops = NULL;
3240         dev->rx_pkt_burst = NULL;
3241         dev->tx_pkt_burst = NULL;
3242
3243         /* disable uio/vfio intr */
3244         rte_intr_disable(intr_handle);
3245
3246         /*PF/VF has different interrupt handling mechanism */
3247         if (hw->mac.type == fm10k_mac_pf) {
3248                 /* disable interrupt */
3249                 fm10k_dev_disable_intr_pf(dev);
3250
3251                 /* unregister callback func to eal lib */
3252                 rte_intr_callback_unregister(intr_handle,
3253                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3254         } else {
3255                 /* disable interrupt */
3256                 fm10k_dev_disable_intr_vf(dev);
3257
3258                 rte_intr_callback_unregister(intr_handle,
3259                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3260         }
3261
3262         /* free mac memory */
3263         if (dev->data->mac_addrs) {
3264                 rte_free(dev->data->mac_addrs);
3265                 dev->data->mac_addrs = NULL;
3266         }
3267
3268         memset(hw, 0, sizeof(*hw));
3269
3270         return 0;
3271 }
3272
3273 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3274         struct rte_pci_device *pci_dev)
3275 {
3276         return rte_eth_dev_pci_generic_probe(pci_dev,
3277                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3278 }
3279
3280 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3281 {
3282         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3283 }
3284
3285 /*
3286  * The set of PCI devices this driver supports. This driver will enable both PF
3287  * and SRIOV-VF devices.
3288  */
3289 static const struct rte_pci_id pci_id_fm10k_map[] = {
3290         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3291         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3292         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3293         { .vendor_id = 0, /* sentinel */ },
3294 };
3295
3296 static struct rte_pci_driver rte_pmd_fm10k = {
3297         .id_table = pci_id_fm10k_map,
3298         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3299                      RTE_PCI_DRV_IOVA_AS_VA,
3300         .probe = eth_fm10k_pci_probe,
3301         .remove = eth_fm10k_pci_remove,
3302 };
3303
3304 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3305 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3306 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3307
3308 RTE_INIT(fm10k_init_log);
3309 static void
3310 fm10k_init_log(void)
3311 {
3312         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3313         if (fm10k_logtype_init >= 0)
3314                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3315         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3316         if (fm10k_logtype_driver >= 0)
3317                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3318 }