drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 __rte_weak int
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 __rte_weak uint16_t
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 __rte_weak uint16_t
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 __rte_weak int
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 __rte_weak void
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 __rte_weak void
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 __rte_weak int
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 __rte_weak uint16_t
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static void
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS)
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930 }
 931
 932 static void
 933 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 934 {
 935         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 936         uint8_t mode;
 937         int status;
 938
 939         PMD_INIT_FUNC_TRACE();
 940
 941         /* Return if it didn't acquire valid glort range */
 942         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 943                 return;
 944
 945         if (dev->data->all_multicast == 1)
 946                 mode = FM10K_XCAST_MODE_ALLMULTI;
 947         else
 948                 mode = FM10K_XCAST_MODE_NONE;
 949
 950         fm10k_mbx_lock(hw);
 951         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 952                                 mode);
 953         fm10k_mbx_unlock(hw);
 954
 955         if (status != FM10K_SUCCESS)
 956                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 957 }
 958
 959 static void
 960 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 961 {
 962         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 963         int status;
 964
 965         PMD_INIT_FUNC_TRACE();
 966
 967         /* Return if it didn't acquire valid glort range */
 968         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 969                 return;
 970
 971         /* If promiscuous mode is enabled, it doesn't make sense to enable
 972          * allmulticast and disable promiscuous since fm10k only can select
 973          * one of the modes.
 974          */
 975         if (dev->data->promiscuous) {
 976                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 977                         "needn't enable allmulticast");
 978                 return;
 979         }
 980
 981         fm10k_mbx_lock(hw);
 982         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 983                                 FM10K_XCAST_MODE_ALLMULTI);
 984         fm10k_mbx_unlock(hw);
 985
 986         if (status != FM10K_SUCCESS)
 987                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 988 }
 989
 990 static void
 991 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 992 {
 993         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 994         int status;
 995
 996         PMD_INIT_FUNC_TRACE();
 997
 998         /* Return if it didn't acquire valid glort range */
 999         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1000                 return;
1001
1002         if (dev->data->promiscuous) {
1003                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1004                         "since promisc mode is enabled");
1005                 return;
1006         }
1007
1008         fm10k_mbx_lock(hw);
1009         /* Change mode to unicast mode */
1010         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1011                                 FM10K_XCAST_MODE_NONE);
1012         fm10k_mbx_unlock(hw);
1013
1014         if (status != FM10K_SUCCESS)
1015                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1016 }
1017
1018 static void
1019 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1020 {
1021         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1022         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1023         uint16_t nb_queue_pools;
1024         struct fm10k_macvlan_filter_info *macvlan;
1025
1026         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1027         nb_queue_pools = macvlan->nb_queue_pools;
1028         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1029         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1030
1031         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1032         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1033         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1034                         hw->mac.dglort_map;
1035         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1036         /* Configure VMDQ/RSS DGlort Decoder */
1037         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1038
1039         /* Flow Director configurations, only queue number is valid. */
1040         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1041         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1042                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1043         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1044         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1045
1046         /* Invalidate all other GLORT entries */
1047         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1048                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1049                                 FM10K_DGLORTMAP_NONE);
1050 }
1051
1052 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1053 static int
1054 fm10k_dev_start(struct rte_eth_dev *dev)
1055 {
1056         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1057         int i, diag;
1058
1059         PMD_INIT_FUNC_TRACE();
1060
1061         /* stop, init, then start the hw */
1062         diag = fm10k_stop_hw(hw);
1063         if (diag != FM10K_SUCCESS) {
1064                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1065                 return -EIO;
1066         }
1067
1068         diag = fm10k_init_hw(hw);
1069         if (diag != FM10K_SUCCESS) {
1070                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1071                 return -EIO;
1072         }
1073
1074         diag = fm10k_start_hw(hw);
1075         if (diag != FM10K_SUCCESS) {
1076                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1077                 return -EIO;
1078         }
1079
1080         diag = fm10k_dev_tx_init(dev);
1081         if (diag) {
1082                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1083                 return diag;
1084         }
1085
1086         if (fm10k_dev_rxq_interrupt_setup(dev))
1087                 return -EIO;
1088
1089         diag = fm10k_dev_rx_init(dev);
1090         if (diag) {
1091                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1092                 return diag;
1093         }
1094
1095         if (hw->mac.type == fm10k_mac_pf)
1096                 fm10k_dev_dglort_map_configure(dev);
1097
1098         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1099                 struct fm10k_rx_queue *rxq;
1100                 rxq = dev->data->rx_queues[i];
1101
1102                 if (rxq->rx_deferred_start)
1103                         continue;
1104                 diag = fm10k_dev_rx_queue_start(dev, i);
1105                 if (diag != 0) {
1106                         int j;
1107                         for (j = 0; j < i; ++j)
1108                                 rx_queue_clean(dev->data->rx_queues[j]);
1109                         return diag;
1110                 }
1111         }
1112
1113         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1114                 struct fm10k_tx_queue *txq;
1115                 txq = dev->data->tx_queues[i];
1116
1117                 if (txq->tx_deferred_start)
1118                         continue;
1119                 diag = fm10k_dev_tx_queue_start(dev, i);
1120                 if (diag != 0) {
1121                         int j;
1122                         for (j = 0; j < i; ++j)
1123                                 tx_queue_clean(dev->data->tx_queues[j]);
1124                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1125                                 rx_queue_clean(dev->data->rx_queues[j]);
1126                         return diag;
1127                 }
1128         }
1129
1130         /* Update default vlan when not in VMDQ mode */
1131         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1132                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1133
1134         fm10k_link_update(dev, 0);
1135
1136         return 0;
1137 }
1138
1139 static void
1140 fm10k_dev_stop(struct rte_eth_dev *dev)
1141 {
1142         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1143         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1144         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1145         int i;
1146
1147         PMD_INIT_FUNC_TRACE();
1148
1149         if (dev->data->tx_queues)
1150                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1151                         fm10k_dev_tx_queue_stop(dev, i);
1152
1153         if (dev->data->rx_queues)
1154                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1155                         fm10k_dev_rx_queue_stop(dev, i);
1156
1157         /* Disable datapath event */
1158         if (rte_intr_dp_is_en(intr_handle)) {
1159                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1160                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1161                                 3 << FM10K_RXINT_TIMER_SHIFT);
1162                         if (hw->mac.type == fm10k_mac_pf)
1163                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1164                                         FM10K_ITR_MASK_SET);
1165                         else
1166                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1167                                         FM10K_ITR_MASK_SET);
1168                 }
1169         }
1170         /* Clean datapath event and queue/vec mapping */
1171         rte_intr_efd_disable(intr_handle);
1172         rte_free(intr_handle->intr_vec);
1173         intr_handle->intr_vec = NULL;
1174 }
1175
1176 static void
1177 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1178 {
1179         int i;
1180
1181         PMD_INIT_FUNC_TRACE();
1182
1183         if (dev->data->tx_queues) {
1184                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1185                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1186
1187                         tx_queue_free(txq);
1188                 }
1189         }
1190
1191         if (dev->data->rx_queues) {
1192                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1193                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1194         }
1195 }
1196
1197 static void
1198 fm10k_dev_close(struct rte_eth_dev *dev)
1199 {
1200         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1201
1202         PMD_INIT_FUNC_TRACE();
1203
1204         fm10k_mbx_lock(hw);
1205         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1206                 MAX_LPORT_NUM, false);
1207         fm10k_mbx_unlock(hw);
1208
1209         /* allow 100ms for device to quiesce */
1210         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1211
1212         /* Stop mailbox service first */
1213         fm10k_close_mbx_service(hw);
1214         fm10k_dev_stop(dev);
1215         fm10k_dev_queue_release(dev);
1216         fm10k_stop_hw(hw);
1217 }
1218
1219 static int
1220 fm10k_link_update(struct rte_eth_dev *dev,
1221         __rte_unused int wait_to_complete)
1222 {
1223         struct fm10k_dev_info *dev_info =
1224                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1225         PMD_INIT_FUNC_TRACE();
1226
1227         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1228         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1229         dev->data->dev_link.link_status =
1230                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1231         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1232
1233         return 0;
1234 }
1235
1236 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1237         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1238 {
1239         unsigned i, q;
1240         unsigned count = 0;
1241
1242         if (xstats_names != NULL) {
1243                 /* Note: limit checked in rte_eth_xstats_names() */
1244
1245                 /* Global stats */
1246                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1247                         snprintf(xstats_names[count].name,
1248                                 sizeof(xstats_names[count].name),
1249                                 "%s", fm10k_hw_stats_strings[count].name);
1250                         count++;
1251                 }
1252
1253                 /* PF queue stats */
1254                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1255                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1256                                 snprintf(xstats_names[count].name,
1257                                         sizeof(xstats_names[count].name),
1258                                         "rx_q%u_%s", q,
1259                                         fm10k_hw_stats_rx_q_strings[i].name);
1260                                 count++;
1261                         }
1262                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1263                                 snprintf(xstats_names[count].name,
1264                                         sizeof(xstats_names[count].name),
1265                                         "tx_q%u_%s", q,
1266                                         fm10k_hw_stats_tx_q_strings[i].name);
1267                                 count++;
1268                         }
1269                 }
1270         }
1271         return FM10K_NB_XSTATS;
1272 }
1273
1274 static int
1275 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1276                  unsigned n)
1277 {
1278         struct fm10k_hw_stats *hw_stats =
1279                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1280         unsigned i, q, count = 0;
1281
1282         if (n < FM10K_NB_XSTATS)
1283                 return FM10K_NB_XSTATS;
1284
1285         /* Global stats */
1286         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1287                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1288                         fm10k_hw_stats_strings[count].offset);
1289                 xstats[count].id = count;
1290                 count++;
1291         }
1292
1293         /* PF queue stats */
1294         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1295                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1296                         xstats[count].value =
1297                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1298                                 fm10k_hw_stats_rx_q_strings[i].offset);
1299                         xstats[count].id = count;
1300                         count++;
1301                 }
1302                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1303                         xstats[count].value =
1304                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1305                                 fm10k_hw_stats_tx_q_strings[i].offset);
1306                         xstats[count].id = count;
1307                         count++;
1308                 }
1309         }
1310
1311         return FM10K_NB_XSTATS;
1312 }
1313
1314 static int
1315 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1316 {
1317         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1318         struct fm10k_hw *hw =
1319                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1320         struct fm10k_hw_stats *hw_stats =
1321                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1322         int i;
1323
1324         PMD_INIT_FUNC_TRACE();
1325
1326         fm10k_update_hw_stats(hw, hw_stats);
1327
1328         ipackets = opackets = ibytes = obytes = imissed = 0;
1329         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1330                 (i < hw->mac.max_queues); ++i) {
1331                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1332                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1333                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1334                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1335                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1336                 ipackets += stats->q_ipackets[i];
1337                 opackets += stats->q_opackets[i];
1338                 ibytes   += stats->q_ibytes[i];
1339                 obytes   += stats->q_obytes[i];
1340                 imissed  += stats->q_errors[i];
1341         }
1342         stats->ipackets = ipackets;
1343         stats->opackets = opackets;
1344         stats->ibytes = ibytes;
1345         stats->obytes = obytes;
1346         stats->imissed = imissed;
1347         return 0;
1348 }
1349
1350 static void
1351 fm10k_stats_reset(struct rte_eth_dev *dev)
1352 {
1353         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1354         struct fm10k_hw_stats *hw_stats =
1355                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1356
1357         PMD_INIT_FUNC_TRACE();
1358
1359         memset(hw_stats, 0, sizeof(*hw_stats));
1360         fm10k_rebind_hw_stats(hw, hw_stats);
1361 }
1362
1363 static int
1364 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1365         struct rte_eth_dev_info *dev_info)
1366 {
1367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1369
1370         PMD_INIT_FUNC_TRACE();
1371
1372         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1373         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1374         dev_info->max_rx_queues      = hw->mac.max_queues;
1375         dev_info->max_tx_queues      = hw->mac.max_queues;
1376         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1377         dev_info->max_hash_mac_addrs = 0;
1378         dev_info->max_vfs            = pdev->max_vfs;
1379         dev_info->vmdq_pool_base     = 0;
1380         dev_info->vmdq_queue_base    = 0;
1381         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1382         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1383         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1384         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1385                                     dev_info->rx_queue_offload_capa;
1386         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1387         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1388                                     dev_info->tx_queue_offload_capa;
1389
1390         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1391         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1392         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1393                                         ETH_RSS_IPV6 |
1394                                         ETH_RSS_IPV6_EX |
1395                                         ETH_RSS_NONFRAG_IPV4_TCP |
1396                                         ETH_RSS_NONFRAG_IPV6_TCP |
1397                                         ETH_RSS_IPV6_TCP_EX |
1398                                         ETH_RSS_NONFRAG_IPV4_UDP |
1399                                         ETH_RSS_NONFRAG_IPV6_UDP |
1400                                         ETH_RSS_IPV6_UDP_EX;
1401
1402         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1403                 .rx_thresh = {
1404                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1405                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1406                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1407                 },
1408                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1409                 .rx_drop_en = 0,
1410                 .offloads = 0,
1411         };
1412
1413         dev_info->default_txconf = (struct rte_eth_txconf) {
1414                 .tx_thresh = {
1415                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1416                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1417                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1418                 },
1419                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1420                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1421                 .offloads = 0,
1422         };
1423
1424         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1425                 .nb_max = FM10K_MAX_RX_DESC,
1426                 .nb_min = FM10K_MIN_RX_DESC,
1427                 .nb_align = FM10K_MULT_RX_DESC,
1428         };
1429
1430         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1431                 .nb_max = FM10K_MAX_TX_DESC,
1432                 .nb_min = FM10K_MIN_TX_DESC,
1433                 .nb_align = FM10K_MULT_TX_DESC,
1434                 .nb_seg_max = FM10K_TX_MAX_SEG,
1435                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1436         };
1437
1438         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1439                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1440                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1441
1442         return 0;
1443 }
1444
1445 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1446 static const uint32_t *
1447 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1448 {
1449         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1450             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1451                 static uint32_t ptypes[] = {
1452                         /* refers to rx_desc_to_ol_flags() */
1453                         RTE_PTYPE_L2_ETHER,
1454                         RTE_PTYPE_L3_IPV4,
1455                         RTE_PTYPE_L3_IPV4_EXT,
1456                         RTE_PTYPE_L3_IPV6,
1457                         RTE_PTYPE_L3_IPV6_EXT,
1458                         RTE_PTYPE_L4_TCP,
1459                         RTE_PTYPE_L4_UDP,
1460                         RTE_PTYPE_UNKNOWN
1461                 };
1462
1463                 return ptypes;
1464         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1465                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1466                 static uint32_t ptypes_vec[] = {
1467                         /* refers to fm10k_desc_to_pktype_v() */
1468                         RTE_PTYPE_L3_IPV4,
1469                         RTE_PTYPE_L3_IPV4_EXT,
1470                         RTE_PTYPE_L3_IPV6,
1471                         RTE_PTYPE_L3_IPV6_EXT,
1472                         RTE_PTYPE_L4_TCP,
1473                         RTE_PTYPE_L4_UDP,
1474                         RTE_PTYPE_TUNNEL_GENEVE,
1475                         RTE_PTYPE_TUNNEL_NVGRE,
1476                         RTE_PTYPE_TUNNEL_VXLAN,
1477                         RTE_PTYPE_TUNNEL_GRE,
1478                         RTE_PTYPE_UNKNOWN
1479                 };
1480
1481                 return ptypes_vec;
1482         }
1483
1484         return NULL;
1485 }
1486 #else
1487 static const uint32_t *
1488 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1489 {
1490         return NULL;
1491 }
1492 #endif
1493
1494 static int
1495 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1496 {
1497         s32 result;
1498         uint16_t mac_num = 0;
1499         uint32_t vid_idx, vid_bit, mac_index;
1500         struct fm10k_hw *hw;
1501         struct fm10k_macvlan_filter_info *macvlan;
1502         struct rte_eth_dev_data *data = dev->data;
1503
1504         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1505         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1506
1507         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1508                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1509                 return -EINVAL;
1510         }
1511
1512         if (vlan_id > ETH_VLAN_ID_MAX) {
1513                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1514                 return -EINVAL;
1515         }
1516
1517         vid_idx = FM10K_VFTA_IDX(vlan_id);
1518         vid_bit = FM10K_VFTA_BIT(vlan_id);
1519         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1520         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1521                 return 0;
1522         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1523         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1524                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1525                         "in the VLAN filter table");
1526                 return -EINVAL;
1527         }
1528
1529         fm10k_mbx_lock(hw);
1530         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1531         fm10k_mbx_unlock(hw);
1532         if (result != FM10K_SUCCESS) {
1533                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1534                 return -EIO;
1535         }
1536
1537         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1538                         (result == FM10K_SUCCESS); mac_index++) {
1539                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1540                         continue;
1541                 if (mac_num > macvlan->mac_num - 1) {
1542                         PMD_INIT_LOG(ERR, "MAC address number "
1543                                         "not match");
1544                         break;
1545                 }
1546                 fm10k_mbx_lock(hw);
1547                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1548                         data->mac_addrs[mac_index].addr_bytes,
1549                         vlan_id, on, 0);
1550                 fm10k_mbx_unlock(hw);
1551                 mac_num++;
1552         }
1553         if (result != FM10K_SUCCESS) {
1554                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1555                 return -EIO;
1556         }
1557
1558         if (on) {
1559                 macvlan->vlan_num++;
1560                 macvlan->vfta[vid_idx] |= vid_bit;
1561         } else {
1562                 macvlan->vlan_num--;
1563                 macvlan->vfta[vid_idx] &= ~vid_bit;
1564         }
1565         return 0;
1566 }
1567
1568 static int
1569 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1570 {
1571         if (mask & ETH_VLAN_STRIP_MASK) {
1572                 if (!(dev->data->dev_conf.rxmode.offloads &
1573                         DEV_RX_OFFLOAD_VLAN_STRIP))
1574                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1575                                         "always on in fm10k");
1576         }
1577
1578         if (mask & ETH_VLAN_EXTEND_MASK) {
1579                 if (dev->data->dev_conf.rxmode.offloads &
1580                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1581                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1582                                         "supported in fm10k");
1583         }
1584
1585         if (mask & ETH_VLAN_FILTER_MASK) {
1586                 if (!(dev->data->dev_conf.rxmode.offloads &
1587                         DEV_RX_OFFLOAD_VLAN_FILTER))
1588                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1589         }
1590
1591         return 0;
1592 }
1593
1594 /* Add/Remove a MAC address, and update filters to main VSI */
1595 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1596                 const u8 *mac, bool add, uint32_t pool)
1597 {
1598         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1599         struct fm10k_macvlan_filter_info *macvlan;
1600         uint32_t i, j, k;
1601
1602         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1603
1604         if (pool != MAIN_VSI_POOL_NUMBER) {
1605                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1606                         "mac to pool %u", pool);
1607                 return;
1608         }
1609         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1610                 if (!macvlan->vfta[j])
1611                         continue;
1612                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1613                         if (!(macvlan->vfta[j] & (1 << k)))
1614                                 continue;
1615                         if (i + 1 > macvlan->vlan_num) {
1616                                 PMD_INIT_LOG(ERR, "vlan number not match");
1617                                 return;
1618                         }
1619                         fm10k_mbx_lock(hw);
1620                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1621                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1622                         fm10k_mbx_unlock(hw);
1623                         i++;
1624                 }
1625         }
1626 }
1627
1628 /* Add/Remove a MAC address, and update filters to VMDQ */
1629 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1630                 const u8 *mac, bool add, uint32_t pool)
1631 {
1632         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1633         struct fm10k_macvlan_filter_info *macvlan;
1634         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1635         uint32_t i;
1636
1637         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1638         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1639
1640         if (pool > macvlan->nb_queue_pools) {
1641                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1642                         " Max pool is %u",
1643                         pool, macvlan->nb_queue_pools);
1644                 return;
1645         }
1646         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1647                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1648                         continue;
1649                 fm10k_mbx_lock(hw);
1650                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1651                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1652                 fm10k_mbx_unlock(hw);
1653         }
1654 }
1655
1656 /* Add/Remove a MAC address, and update filters */
1657 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1658                 const u8 *mac, bool add, uint32_t pool)
1659 {
1660         struct fm10k_macvlan_filter_info *macvlan;
1661
1662         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1663
1664         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1665                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1666         else
1667                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1668
1669         if (add)
1670                 macvlan->mac_num++;
1671         else
1672                 macvlan->mac_num--;
1673 }
1674
1675 /* Add a MAC address, and update filters */
1676 static int
1677 fm10k_macaddr_add(struct rte_eth_dev *dev,
1678                 struct rte_ether_addr *mac_addr,
1679                 uint32_t index,
1680                 uint32_t pool)
1681 {
1682         struct fm10k_macvlan_filter_info *macvlan;
1683
1684         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1685         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1686         macvlan->mac_vmdq_id[index] = pool;
1687         return 0;
1688 }
1689
1690 /* Remove a MAC address, and update filters */
1691 static void
1692 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1693 {
1694         struct rte_eth_dev_data *data = dev->data;
1695         struct fm10k_macvlan_filter_info *macvlan;
1696
1697         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1698         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1699                         FALSE, macvlan->mac_vmdq_id[index]);
1700         macvlan->mac_vmdq_id[index] = 0;
1701 }
1702
1703 static inline int
1704 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1705 {
1706         if ((request < min) || (request > max) || ((request % mult) != 0))
1707                 return -1;
1708         else
1709                 return 0;
1710 }
1711
1712
1713 static inline int
1714 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1715 {
1716         if ((request < min) || (request > max) || ((div % request) != 0))
1717                 return -1;
1718         else
1719                 return 0;
1720 }
1721
1722 static inline int
1723 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1724 {
1725         uint16_t rx_free_thresh;
1726
1727         if (conf->rx_free_thresh == 0)
1728                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1729         else
1730                 rx_free_thresh = conf->rx_free_thresh;
1731
1732         /* make sure the requested threshold satisfies the constraints */
1733         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1734                         FM10K_RX_FREE_THRESH_MAX(q),
1735                         FM10K_RX_FREE_THRESH_DIV(q),
1736                         rx_free_thresh)) {
1737                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1738                         "less than or equal to %u, "
1739                         "greater than or equal to %u, "
1740                         "and a divisor of %u",
1741                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1742                         FM10K_RX_FREE_THRESH_MIN(q),
1743                         FM10K_RX_FREE_THRESH_DIV(q));
1744                 return -EINVAL;
1745         }
1746
1747         q->alloc_thresh = rx_free_thresh;
1748         q->drop_en = conf->rx_drop_en;
1749         q->rx_deferred_start = conf->rx_deferred_start;
1750
1751         return 0;
1752 }
1753
1754 /*
1755  * Hardware requires specific alignment for Rx packet buffers. At
1756  * least one of the following two conditions must be satisfied.
1757  *  1. Address is 512B aligned
1758  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1759  *
1760  * As such, the driver may need to adjust the DMA address within the
1761  * buffer by up to 512B.
1762  *
1763  * return 1 if the element size is valid, otherwise return 0.
1764  */
1765 static int
1766 mempool_element_size_valid(struct rte_mempool *mp)
1767 {
1768         uint32_t min_size;
1769
1770         /* elt_size includes mbuf header and headroom */
1771         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1772                         RTE_PKTMBUF_HEADROOM;
1773
1774         /* account for up to 512B of alignment */
1775         min_size -= FM10K_RX_DATABUF_ALIGN;
1776
1777         /* sanity check for overflow */
1778         if (min_size > mp->elt_size)
1779                 return 0;
1780
1781         /* size is valid */
1782         return 1;
1783 }
1784
1785 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1786 {
1787         RTE_SET_USED(dev);
1788
1789         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1790 }
1791
1792 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1793 {
1794         RTE_SET_USED(dev);
1795
1796         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1797                            DEV_RX_OFFLOAD_VLAN_FILTER |
1798                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1799                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1800                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1801                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1802                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1803 }
1804
1805 static int
1806 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1807         uint16_t nb_desc, unsigned int socket_id,
1808         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1809 {
1810         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1811         struct fm10k_dev_info *dev_info =
1812                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1813         struct fm10k_rx_queue *q;
1814         const struct rte_memzone *mz;
1815         uint64_t offloads;
1816
1817         PMD_INIT_FUNC_TRACE();
1818
1819         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1820
1821         /* make sure the mempool element size can account for alignment. */
1822         if (!mempool_element_size_valid(mp)) {
1823                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1824                 return -EINVAL;
1825         }
1826
1827         /* make sure a valid number of descriptors have been requested */
1828         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1829                                 FM10K_MULT_RX_DESC, nb_desc)) {
1830                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1831                         "less than or equal to %"PRIu32", "
1832                         "greater than or equal to %u, "
1833                         "and a multiple of %u",
1834                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1835                         FM10K_MULT_RX_DESC);
1836                 return -EINVAL;
1837         }
1838
1839         /*
1840          * if this queue existed already, free the associated memory. The
1841          * queue cannot be reused in case we need to allocate memory on
1842          * different socket than was previously used.
1843          */
1844         if (dev->data->rx_queues[queue_id] != NULL) {
1845                 rx_queue_free(dev->data->rx_queues[queue_id]);
1846                 dev->data->rx_queues[queue_id] = NULL;
1847         }
1848
1849         /* allocate memory for the queue structure */
1850         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1851                                 socket_id);
1852         if (q == NULL) {
1853                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1854                 return -ENOMEM;
1855         }
1856
1857         /* setup queue */
1858         q->mp = mp;
1859         q->nb_desc = nb_desc;
1860         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1861         q->port_id = dev->data->port_id;
1862         q->queue_id = queue_id;
1863         q->tail_ptr = (volatile uint32_t *)
1864                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1865         q->offloads = offloads;
1866         if (handle_rxconf(q, conf))
1867                 return -EINVAL;
1868
1869         /* allocate memory for the software ring */
1870         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1871                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1872                         RTE_CACHE_LINE_SIZE, socket_id);
1873         if (q->sw_ring == NULL) {
1874                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1875                 rte_free(q);
1876                 return -ENOMEM;
1877         }
1878
1879         /*
1880          * allocate memory for the hardware descriptor ring. A memzone large
1881          * enough to hold the maximum ring size is requested to allow for
1882          * resizing in later calls to the queue setup function.
1883          */
1884         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1885                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1886                                       socket_id);
1887         if (mz == NULL) {
1888                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1889                 rte_free(q->sw_ring);
1890                 rte_free(q);
1891                 return -ENOMEM;
1892         }
1893         q->hw_ring = mz->addr;
1894         q->hw_ring_phys_addr = mz->iova;
1895
1896         /* Check if number of descs satisfied Vector requirement */
1897         if (!rte_is_power_of_2(nb_desc)) {
1898                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1899                                     "preconditions - canceling the feature for "
1900                                     "the whole port[%d]",
1901                              q->queue_id, q->port_id);
1902                 dev_info->rx_vec_allowed = false;
1903         } else
1904                 fm10k_rxq_vec_setup(q);
1905
1906         dev->data->rx_queues[queue_id] = q;
1907         return 0;
1908 }
1909
1910 static void
1911 fm10k_rx_queue_release(void *queue)
1912 {
1913         PMD_INIT_FUNC_TRACE();
1914
1915         rx_queue_free(queue);
1916 }
1917
1918 static inline int
1919 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1920 {
1921         uint16_t tx_free_thresh;
1922         uint16_t tx_rs_thresh;
1923
1924         /* constraint MACROs require that tx_free_thresh is configured
1925          * before tx_rs_thresh */
1926         if (conf->tx_free_thresh == 0)
1927                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1928         else
1929                 tx_free_thresh = conf->tx_free_thresh;
1930
1931         /* make sure the requested threshold satisfies the constraints */
1932         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1933                         FM10K_TX_FREE_THRESH_MAX(q),
1934                         FM10K_TX_FREE_THRESH_DIV(q),
1935                         tx_free_thresh)) {
1936                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1937                         "less than or equal to %u, "
1938                         "greater than or equal to %u, "
1939                         "and a divisor of %u",
1940                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1941                         FM10K_TX_FREE_THRESH_MIN(q),
1942                         FM10K_TX_FREE_THRESH_DIV(q));
1943                 return -EINVAL;
1944         }
1945
1946         q->free_thresh = tx_free_thresh;
1947
1948         if (conf->tx_rs_thresh == 0)
1949                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1950         else
1951                 tx_rs_thresh = conf->tx_rs_thresh;
1952
1953         q->tx_deferred_start = conf->tx_deferred_start;
1954
1955         /* make sure the requested threshold satisfies the constraints */
1956         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1957                         FM10K_TX_RS_THRESH_MAX(q),
1958                         FM10K_TX_RS_THRESH_DIV(q),
1959                         tx_rs_thresh)) {
1960                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1961                         "less than or equal to %u, "
1962                         "greater than or equal to %u, "
1963                         "and a divisor of %u",
1964                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1965                         FM10K_TX_RS_THRESH_MIN(q),
1966                         FM10K_TX_RS_THRESH_DIV(q));
1967                 return -EINVAL;
1968         }
1969
1970         q->rs_thresh = tx_rs_thresh;
1971
1972         return 0;
1973 }
1974
1975 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1976 {
1977         RTE_SET_USED(dev);
1978
1979         return 0;
1980 }
1981
1982 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1983 {
1984         RTE_SET_USED(dev);
1985
1986         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1987                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1988                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1989                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1990                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1991                           DEV_TX_OFFLOAD_TCP_TSO);
1992 }
1993
1994 static int
1995 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1996         uint16_t nb_desc, unsigned int socket_id,
1997         const struct rte_eth_txconf *conf)
1998 {
1999         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2000         struct fm10k_tx_queue *q;
2001         const struct rte_memzone *mz;
2002         uint64_t offloads;
2003
2004         PMD_INIT_FUNC_TRACE();
2005
2006         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2007
2008         /* make sure a valid number of descriptors have been requested */
2009         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2010                                 FM10K_MULT_TX_DESC, nb_desc)) {
2011                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2012                         "less than or equal to %"PRIu32", "
2013                         "greater than or equal to %u, "
2014                         "and a multiple of %u",
2015                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2016                         FM10K_MULT_TX_DESC);
2017                 return -EINVAL;
2018         }
2019
2020         /*
2021          * if this queue existed already, free the associated memory. The
2022          * queue cannot be reused in case we need to allocate memory on
2023          * different socket than was previously used.
2024          */
2025         if (dev->data->tx_queues[queue_id] != NULL) {
2026                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2027
2028                 tx_queue_free(txq);
2029                 dev->data->tx_queues[queue_id] = NULL;
2030         }
2031
2032         /* allocate memory for the queue structure */
2033         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2034                                 socket_id);
2035         if (q == NULL) {
2036                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2037                 return -ENOMEM;
2038         }
2039
2040         /* setup queue */
2041         q->nb_desc = nb_desc;
2042         q->port_id = dev->data->port_id;
2043         q->queue_id = queue_id;
2044         q->offloads = offloads;
2045         q->ops = &def_txq_ops;
2046         q->tail_ptr = (volatile uint32_t *)
2047                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2048         if (handle_txconf(q, conf))
2049                 return -EINVAL;
2050
2051         /* allocate memory for the software ring */
2052         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2053                                         nb_desc * sizeof(struct rte_mbuf *),
2054                                         RTE_CACHE_LINE_SIZE, socket_id);
2055         if (q->sw_ring == NULL) {
2056                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2057                 rte_free(q);
2058                 return -ENOMEM;
2059         }
2060
2061         /*
2062          * allocate memory for the hardware descriptor ring. A memzone large
2063          * enough to hold the maximum ring size is requested to allow for
2064          * resizing in later calls to the queue setup function.
2065          */
2066         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2067                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2068                                       socket_id);
2069         if (mz == NULL) {
2070                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2071                 rte_free(q->sw_ring);
2072                 rte_free(q);
2073                 return -ENOMEM;
2074         }
2075         q->hw_ring = mz->addr;
2076         q->hw_ring_phys_addr = mz->iova;
2077
2078         /*
2079          * allocate memory for the RS bit tracker. Enough slots to hold the
2080          * descriptor index for each RS bit needing to be set are required.
2081          */
2082         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2083                                 ((nb_desc + 1) / q->rs_thresh) *
2084                                 sizeof(uint16_t),
2085                                 RTE_CACHE_LINE_SIZE, socket_id);
2086         if (q->rs_tracker.list == NULL) {
2087                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2088                 rte_free(q->sw_ring);
2089                 rte_free(q);
2090                 return -ENOMEM;
2091         }
2092
2093         dev->data->tx_queues[queue_id] = q;
2094         return 0;
2095 }
2096
2097 static void
2098 fm10k_tx_queue_release(void *queue)
2099 {
2100         struct fm10k_tx_queue *q = queue;
2101         PMD_INIT_FUNC_TRACE();
2102
2103         tx_queue_free(q);
2104 }
2105
2106 static int
2107 fm10k_reta_update(struct rte_eth_dev *dev,
2108                         struct rte_eth_rss_reta_entry64 *reta_conf,
2109                         uint16_t reta_size)
2110 {
2111         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2112         uint16_t i, j, idx, shift;
2113         uint8_t mask;
2114         uint32_t reta;
2115
2116         PMD_INIT_FUNC_TRACE();
2117
2118         if (reta_size > FM10K_MAX_RSS_INDICES) {
2119                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2120                         "(%d) doesn't match the number hardware can supported "
2121                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2122                 return -EINVAL;
2123         }
2124
2125         /*
2126          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2127          * 128-entries in 32 registers
2128          */
2129         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2130                 idx = i / RTE_RETA_GROUP_SIZE;
2131                 shift = i % RTE_RETA_GROUP_SIZE;
2132                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2133                                 BIT_MASK_PER_UINT32);
2134                 if (mask == 0)
2135                         continue;
2136
2137                 reta = 0;
2138                 if (mask != BIT_MASK_PER_UINT32)
2139                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2140
2141                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2142                         if (mask & (0x1 << j)) {
2143                                 if (mask != 0xF)
2144                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2145                                 reta |= reta_conf[idx].reta[shift + j] <<
2146                                                 (CHAR_BIT * j);
2147                         }
2148                 }
2149                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2150         }
2151
2152         return 0;
2153 }
2154
2155 static int
2156 fm10k_reta_query(struct rte_eth_dev *dev,
2157                         struct rte_eth_rss_reta_entry64 *reta_conf,
2158                         uint16_t reta_size)
2159 {
2160         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2161         uint16_t i, j, idx, shift;
2162         uint8_t mask;
2163         uint32_t reta;
2164
2165         PMD_INIT_FUNC_TRACE();
2166
2167         if (reta_size < FM10K_MAX_RSS_INDICES) {
2168                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2169                         "(%d) doesn't match the number hardware can supported "
2170                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2171                 return -EINVAL;
2172         }
2173
2174         /*
2175          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2176          * 128-entries in 32 registers
2177          */
2178         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2179                 idx = i / RTE_RETA_GROUP_SIZE;
2180                 shift = i % RTE_RETA_GROUP_SIZE;
2181                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2182                                 BIT_MASK_PER_UINT32);
2183                 if (mask == 0)
2184                         continue;
2185
2186                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2187                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2188                         if (mask & (0x1 << j))
2189                                 reta_conf[idx].reta[shift + j] = ((reta >>
2190                                         CHAR_BIT * j) & UINT8_MAX);
2191                 }
2192         }
2193
2194         return 0;
2195 }
2196
2197 static int
2198 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2199         struct rte_eth_rss_conf *rss_conf)
2200 {
2201         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2202         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2203         uint32_t mrqc;
2204         uint64_t hf = rss_conf->rss_hf;
2205         int i;
2206
2207         PMD_INIT_FUNC_TRACE();
2208
2209         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2210                                 FM10K_RSSRK_ENTRIES_PER_REG))
2211                 return -EINVAL;
2212
2213         if (hf == 0)
2214                 return -EINVAL;
2215
2216         mrqc = 0;
2217         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2218         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2219         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2220         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2221         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2222         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2223         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2224         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2225         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2226
2227         /* If the mapping doesn't fit any supported, return */
2228         if (mrqc == 0)
2229                 return -EINVAL;
2230
2231         if (key != NULL)
2232                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2233                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2234
2235         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2236
2237         return 0;
2238 }
2239
2240 static int
2241 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2242         struct rte_eth_rss_conf *rss_conf)
2243 {
2244         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2245         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2246         uint32_t mrqc;
2247         uint64_t hf;
2248         int i;
2249
2250         PMD_INIT_FUNC_TRACE();
2251
2252         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2253                                 FM10K_RSSRK_ENTRIES_PER_REG))
2254                 return -EINVAL;
2255
2256         if (key != NULL)
2257                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2258                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2259
2260         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2261         hf = 0;
2262         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2263         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2264         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2265         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2266         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2267         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2268         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2269         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2270         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2271
2272         rss_conf->rss_hf = hf;
2273
2274         return 0;
2275 }
2276
2277 static void
2278 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2279 {
2280         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2281         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2282
2283         /* Bind all local non-queue interrupt to vector 0 */
2284         int_map |= FM10K_MISC_VEC_ID;
2285
2286         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2287         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2288         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2289         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2290         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2291         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2292
2293         /* Enable misc causes */
2294         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2295                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2296                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2297                                 FM10K_EIMR_ENABLE(MAILBOX) |
2298                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2299                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2300                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2301                                 FM10K_EIMR_ENABLE(VFLR));
2302
2303         /* Enable ITR 0 */
2304         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2305                                         FM10K_ITR_MASK_CLEAR);
2306         FM10K_WRITE_FLUSH(hw);
2307 }
2308
2309 static void
2310 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2311 {
2312         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2313         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2314
2315         int_map |= FM10K_MISC_VEC_ID;
2316
2317         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2318         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2319         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2320         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2321         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2322         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2323
2324         /* Disable misc causes */
2325         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2326                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2327                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2328                                 FM10K_EIMR_DISABLE(MAILBOX) |
2329                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2330                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2331                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2332                                 FM10K_EIMR_DISABLE(VFLR));
2333
2334         /* Disable ITR 0 */
2335         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2336         FM10K_WRITE_FLUSH(hw);
2337 }
2338
2339 static void
2340 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2341 {
2342         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2343         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2344
2345         /* Bind all local non-queue interrupt to vector 0 */
2346         int_map |= FM10K_MISC_VEC_ID;
2347
2348         /* Only INT 0 available, other 15 are reserved. */
2349         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2350
2351         /* Enable ITR 0 */
2352         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2353                                         FM10K_ITR_MASK_CLEAR);
2354         FM10K_WRITE_FLUSH(hw);
2355 }
2356
2357 static void
2358 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2359 {
2360         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2361         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2362
2363         int_map |= FM10K_MISC_VEC_ID;
2364
2365         /* Only INT 0 available, other 15 are reserved. */
2366         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2367
2368         /* Disable ITR 0 */
2369         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2370         FM10K_WRITE_FLUSH(hw);
2371 }
2372
2373 static int
2374 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2375 {
2376         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2377         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2378
2379         /* Enable ITR */
2380         if (hw->mac.type == fm10k_mac_pf)
2381                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2382                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2383         else
2384                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2385                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2386         rte_intr_ack(&pdev->intr_handle);
2387         return 0;
2388 }
2389
2390 static int
2391 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2392 {
2393         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2394         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2395
2396         /* Disable ITR */
2397         if (hw->mac.type == fm10k_mac_pf)
2398                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2399                         FM10K_ITR_MASK_SET);
2400         else
2401                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2402                         FM10K_ITR_MASK_SET);
2403         return 0;
2404 }
2405
2406 static int
2407 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2408 {
2409         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2410         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2411         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2412         uint32_t intr_vector, vec;
2413         uint16_t queue_id;
2414         int result = 0;
2415
2416         /* fm10k needs one separate interrupt for mailbox,
2417          * so only drivers which support multiple interrupt vectors
2418          * e.g. vfio-pci can work for fm10k interrupt mode
2419          */
2420         if (!rte_intr_cap_multiple(intr_handle) ||
2421                         dev->data->dev_conf.intr_conf.rxq == 0)
2422                 return result;
2423
2424         intr_vector = dev->data->nb_rx_queues;
2425
2426         /* disable interrupt first */
2427         rte_intr_disable(intr_handle);
2428         if (hw->mac.type == fm10k_mac_pf)
2429                 fm10k_dev_disable_intr_pf(dev);
2430         else
2431                 fm10k_dev_disable_intr_vf(dev);
2432
2433         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2434                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2435                 result = -EIO;
2436         }
2437
2438         if (rte_intr_dp_is_en(intr_handle) && !result) {
2439                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2440                         dev->data->nb_rx_queues * sizeof(int), 0);
2441                 if (intr_handle->intr_vec) {
2442                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2443                                         queue_id < dev->data->nb_rx_queues;
2444                                         queue_id++) {
2445                                 intr_handle->intr_vec[queue_id] = vec;
2446                                 if (vec < intr_handle->nb_efd - 1
2447                                                 + FM10K_RX_VEC_START)
2448                                         vec++;
2449                         }
2450                 } else {
2451                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2452                                 " intr_vec", dev->data->nb_rx_queues);
2453                         rte_intr_efd_disable(intr_handle);
2454                         result = -ENOMEM;
2455                 }
2456         }
2457
2458         if (hw->mac.type == fm10k_mac_pf)
2459                 fm10k_dev_enable_intr_pf(dev);
2460         else
2461                 fm10k_dev_enable_intr_vf(dev);
2462         rte_intr_enable(intr_handle);
2463         hw->mac.ops.update_int_moderator(hw);
2464         return result;
2465 }
2466
2467 static int
2468 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2469 {
2470         struct fm10k_fault fault;
2471         int err;
2472         const char *estr = "Unknown error";
2473
2474         /* Process PCA fault */
2475         if (eicr & FM10K_EICR_PCA_FAULT) {
2476                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2477                 if (err)
2478                         goto error;
2479                 switch (fault.type) {
2480                 case PCA_NO_FAULT:
2481                         estr = "PCA_NO_FAULT"; break;
2482                 case PCA_UNMAPPED_ADDR:
2483                         estr = "PCA_UNMAPPED_ADDR"; break;
2484                 case PCA_BAD_QACCESS_PF:
2485                         estr = "PCA_BAD_QACCESS_PF"; break;
2486                 case PCA_BAD_QACCESS_VF:
2487                         estr = "PCA_BAD_QACCESS_VF"; break;
2488                 case PCA_MALICIOUS_REQ:
2489                         estr = "PCA_MALICIOUS_REQ"; break;
2490                 case PCA_POISONED_TLP:
2491                         estr = "PCA_POISONED_TLP"; break;
2492                 case PCA_TLP_ABORT:
2493                         estr = "PCA_TLP_ABORT"; break;
2494                 default:
2495                         goto error;
2496                 }
2497                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2498                         estr, fault.func ? "VF" : "PF", fault.func,
2499                         fault.address, fault.specinfo);
2500         }
2501
2502         /* Process THI fault */
2503         if (eicr & FM10K_EICR_THI_FAULT) {
2504                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2505                 if (err)
2506                         goto error;
2507                 switch (fault.type) {
2508                 case THI_NO_FAULT:
2509                         estr = "THI_NO_FAULT"; break;
2510                 case THI_MAL_DIS_Q_FAULT:
2511                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2512                 default:
2513                         goto error;
2514                 }
2515                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2516                         estr, fault.func ? "VF" : "PF", fault.func,
2517                         fault.address, fault.specinfo);
2518         }
2519
2520         /* Process FUM fault */
2521         if (eicr & FM10K_EICR_FUM_FAULT) {
2522                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2523                 if (err)
2524                         goto error;
2525                 switch (fault.type) {
2526                 case FUM_NO_FAULT:
2527                         estr = "FUM_NO_FAULT"; break;
2528                 case FUM_UNMAPPED_ADDR:
2529                         estr = "FUM_UNMAPPED_ADDR"; break;
2530                 case FUM_POISONED_TLP:
2531                         estr = "FUM_POISONED_TLP"; break;
2532                 case FUM_BAD_VF_QACCESS:
2533                         estr = "FUM_BAD_VF_QACCESS"; break;
2534                 case FUM_ADD_DECODE_ERR:
2535                         estr = "FUM_ADD_DECODE_ERR"; break;
2536                 case FUM_RO_ERROR:
2537                         estr = "FUM_RO_ERROR"; break;
2538                 case FUM_QPRC_CRC_ERROR:
2539                         estr = "FUM_QPRC_CRC_ERROR"; break;
2540                 case FUM_CSR_TIMEOUT:
2541                         estr = "FUM_CSR_TIMEOUT"; break;
2542                 case FUM_INVALID_TYPE:
2543                         estr = "FUM_INVALID_TYPE"; break;
2544                 case FUM_INVALID_LENGTH:
2545                         estr = "FUM_INVALID_LENGTH"; break;
2546                 case FUM_INVALID_BE:
2547                         estr = "FUM_INVALID_BE"; break;
2548                 case FUM_INVALID_ALIGN:
2549                         estr = "FUM_INVALID_ALIGN"; break;
2550                 default:
2551                         goto error;
2552                 }
2553                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2554                         estr, fault.func ? "VF" : "PF", fault.func,
2555                         fault.address, fault.specinfo);
2556         }
2557
2558         return 0;
2559 error:
2560         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2561         return err;
2562 }
2563
2564 /**
2565  * PF interrupt handler triggered by NIC for handling specific interrupt.
2566  *
2567  * @param handle
2568  *  Pointer to interrupt handle.
2569  * @param param
2570  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2571  *
2572  * @return
2573  *  void
2574  */
2575 static void
2576 fm10k_dev_interrupt_handler_pf(void *param)
2577 {
2578         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2579         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2580         uint32_t cause, status;
2581         struct fm10k_dev_info *dev_info =
2582                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2583         int status_mbx;
2584         s32 err;
2585
2586         if (hw->mac.type != fm10k_mac_pf)
2587                 return;
2588
2589         cause = FM10K_READ_REG(hw, FM10K_EICR);
2590
2591         /* Handle PCI fault cases */
2592         if (cause & FM10K_EICR_FAULT_MASK) {
2593                 PMD_INIT_LOG(ERR, "INT: find fault!");
2594                 fm10k_dev_handle_fault(hw, cause);
2595         }
2596
2597         /* Handle switch up/down */
2598         if (cause & FM10K_EICR_SWITCHNOTREADY)
2599                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2600
2601         if (cause & FM10K_EICR_SWITCHREADY) {
2602                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2603                 if (dev_info->sm_down == 1) {
2604                         fm10k_mbx_lock(hw);
2605
2606                         /* For recreating logical ports */
2607                         status_mbx = hw->mac.ops.update_lport_state(hw,
2608                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2609                         if (status_mbx == FM10K_SUCCESS)
2610                                 PMD_INIT_LOG(INFO,
2611                                         "INT: Recreated Logical port");
2612                         else
2613                                 PMD_INIT_LOG(INFO,
2614                                         "INT: Logical ports weren't recreated");
2615
2616                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2617                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2618                         if (status_mbx != FM10K_SUCCESS)
2619                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2620
2621                         fm10k_mbx_unlock(hw);
2622
2623                         /* first clear the internal SW recording structure */
2624                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2625                                                 ETH_MQ_RX_VMDQ_FLAG))
2626                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2627                                         false);
2628
2629                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2630                                         MAIN_VSI_POOL_NUMBER);
2631
2632                         /*
2633                          * Add default mac address and vlan for the logical
2634                          * ports that have been created, leave to the
2635                          * application to fully recover Rx filtering.
2636                          */
2637                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2638                                         MAIN_VSI_POOL_NUMBER);
2639
2640                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2641                                                 ETH_MQ_RX_VMDQ_FLAG))
2642                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2643                                         true);
2644
2645                         dev_info->sm_down = 0;
2646                         _rte_eth_dev_callback_process(dev,
2647                                         RTE_ETH_EVENT_INTR_LSC,
2648                                         NULL);
2649                 }
2650         }
2651
2652         /* Handle mailbox message */
2653         fm10k_mbx_lock(hw);
2654         err = hw->mbx.ops.process(hw, &hw->mbx);
2655         fm10k_mbx_unlock(hw);
2656
2657         if (err == FM10K_ERR_RESET_REQUESTED) {
2658                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2659                 dev_info->sm_down = 1;
2660                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2661                                 NULL);
2662         }
2663
2664         /* Handle SRAM error */
2665         if (cause & FM10K_EICR_SRAMERROR) {
2666                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2667
2668                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2669                 /* Write to clear pending bits */
2670                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2671
2672                 /* Todo: print out error message after shared code  updates */
2673         }
2674
2675         /* Clear these 3 events if having any */
2676         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2677                  FM10K_EICR_SWITCHREADY;
2678         if (cause)
2679                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2680
2681         /* Re-enable interrupt from device side */
2682         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2683                                         FM10K_ITR_MASK_CLEAR);
2684         /* Re-enable interrupt from host side */
2685         rte_intr_ack(dev->intr_handle);
2686 }
2687
2688 /**
2689  * VF interrupt handler triggered by NIC for handling specific interrupt.
2690  *
2691  * @param handle
2692  *  Pointer to interrupt handle.
2693  * @param param
2694  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2695  *
2696  * @return
2697  *  void
2698  */
2699 static void
2700 fm10k_dev_interrupt_handler_vf(void *param)
2701 {
2702         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2703         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2704         struct fm10k_mbx_info *mbx = &hw->mbx;
2705         struct fm10k_dev_info *dev_info =
2706                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2707         const enum fm10k_mbx_state state = mbx->state;
2708         int status_mbx;
2709
2710         if (hw->mac.type != fm10k_mac_vf)
2711                 return;
2712
2713         /* Handle mailbox message if lock is acquired */
2714         fm10k_mbx_lock(hw);
2715         hw->mbx.ops.process(hw, &hw->mbx);
2716         fm10k_mbx_unlock(hw);
2717
2718         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2719                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2720
2721                 fm10k_mbx_lock(hw);
2722                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2723                                 MAX_LPORT_NUM, 1);
2724                 fm10k_mbx_unlock(hw);
2725
2726                 /* Setting reset flag */
2727                 dev_info->sm_down = 1;
2728                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2729                                 NULL);
2730         }
2731
2732         if (dev_info->sm_down == 1 &&
2733                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2734                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2735                 fm10k_mbx_lock(hw);
2736                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2737                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2738                 if (status_mbx != FM10K_SUCCESS)
2739                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2740                 fm10k_mbx_unlock(hw);
2741
2742                 /* first clear the internal SW recording structure */
2743                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2744                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2745                                 MAIN_VSI_POOL_NUMBER);
2746
2747                 /*
2748                  * Add default mac address and vlan for the logical ports that
2749                  * have been created, leave to the application to fully recover
2750                  * Rx filtering.
2751                  */
2752                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2753                                 MAIN_VSI_POOL_NUMBER);
2754                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2755
2756                 dev_info->sm_down = 0;
2757                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2758                                 NULL);
2759         }
2760
2761         /* Re-enable interrupt from device side */
2762         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2763                                         FM10K_ITR_MASK_CLEAR);
2764         /* Re-enable interrupt from host side */
2765         rte_intr_ack(dev->intr_handle);
2766 }
2767
2768 /* Mailbox message handler in VF */
2769 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2770         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2771         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2772         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2773         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2774 };
2775
2776 static int
2777 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2778 {
2779         int err = 0;
2780
2781         /* Initialize mailbox lock */
2782         fm10k_mbx_initlock(hw);
2783
2784         /* Replace default message handler with new ones */
2785         if (hw->mac.type == fm10k_mac_vf)
2786                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2787
2788         if (err) {
2789                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2790                                 err);
2791                 return err;
2792         }
2793         /* Connect to SM for PF device or PF for VF device */
2794         return hw->mbx.ops.connect(hw, &hw->mbx);
2795 }
2796
2797 static void
2798 fm10k_close_mbx_service(struct fm10k_hw *hw)
2799 {
2800         /* Disconnect from SM for PF device or PF for VF device */
2801         hw->mbx.ops.disconnect(hw, &hw->mbx);
2802 }
2803
2804 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2805         .dev_configure          = fm10k_dev_configure,
2806         .dev_start              = fm10k_dev_start,
2807         .dev_stop               = fm10k_dev_stop,
2808         .dev_close              = fm10k_dev_close,
2809         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2810         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2811         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2812         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2813         .stats_get              = fm10k_stats_get,
2814         .xstats_get             = fm10k_xstats_get,
2815         .xstats_get_names       = fm10k_xstats_get_names,
2816         .stats_reset            = fm10k_stats_reset,
2817         .xstats_reset           = fm10k_stats_reset,
2818         .link_update            = fm10k_link_update,
2819         .dev_infos_get          = fm10k_dev_infos_get,
2820         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2821         .vlan_filter_set        = fm10k_vlan_filter_set,
2822         .vlan_offload_set       = fm10k_vlan_offload_set,
2823         .mac_addr_add           = fm10k_macaddr_add,
2824         .mac_addr_remove        = fm10k_macaddr_remove,
2825         .rx_queue_start         = fm10k_dev_rx_queue_start,
2826         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2827         .tx_queue_start         = fm10k_dev_tx_queue_start,
2828         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2829         .rx_queue_setup         = fm10k_rx_queue_setup,
2830         .rx_queue_release       = fm10k_rx_queue_release,
2831         .tx_queue_setup         = fm10k_tx_queue_setup,
2832         .tx_queue_release       = fm10k_tx_queue_release,
2833         .rx_queue_count         = fm10k_dev_rx_queue_count,
2834         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2835         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2836         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2837         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2838         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2839         .reta_update            = fm10k_reta_update,
2840         .reta_query             = fm10k_reta_query,
2841         .rss_hash_update        = fm10k_rss_hash_update,
2842         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2843 };
2844
2845 static int ftag_check_handler(__rte_unused const char *key,
2846                 const char *value, __rte_unused void *opaque)
2847 {
2848         if (strcmp(value, "1"))
2849                 return -1;
2850
2851         return 0;
2852 }
2853
2854 static int
2855 fm10k_check_ftag(struct rte_devargs *devargs)
2856 {
2857         struct rte_kvargs *kvlist;
2858         const char *ftag_key = "enable_ftag";
2859
2860         if (devargs == NULL)
2861                 return 0;
2862
2863         kvlist = rte_kvargs_parse(devargs->args, NULL);
2864         if (kvlist == NULL)
2865                 return 0;
2866
2867         if (!rte_kvargs_count(kvlist, ftag_key)) {
2868                 rte_kvargs_free(kvlist);
2869                 return 0;
2870         }
2871         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2872         if (rte_kvargs_process(kvlist, ftag_key,
2873                                 ftag_check_handler, NULL) < 0) {
2874                 rte_kvargs_free(kvlist);
2875                 return 0;
2876         }
2877         rte_kvargs_free(kvlist);
2878
2879         return 1;
2880 }
2881
2882 static uint16_t
2883 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2884                     uint16_t nb_pkts)
2885 {
2886         uint16_t nb_tx = 0;
2887         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2888
2889         while (nb_pkts) {
2890                 uint16_t ret, num;
2891
2892                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2893                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2894                                                  num);
2895                 nb_tx += ret;
2896                 nb_pkts -= ret;
2897                 if (ret < num)
2898                         break;
2899         }
2900
2901         return nb_tx;
2902 }
2903
2904 static void __attribute__((cold))
2905 fm10k_set_tx_function(struct rte_eth_dev *dev)
2906 {
2907         struct fm10k_tx_queue *txq;
2908         int i;
2909         int use_sse = 1;
2910         uint16_t tx_ftag_en = 0;
2911
2912         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2913                 /* primary process has set the ftag flag and offloads */
2914                 txq = dev->data->tx_queues[0];
2915                 if (fm10k_tx_vec_condition_check(txq)) {
2916                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2917                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2918                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2919                 } else {
2920                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2921                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2922                         dev->tx_pkt_prepare = NULL;
2923                 }
2924                 return;
2925         }
2926
2927         if (fm10k_check_ftag(dev->device->devargs))
2928                 tx_ftag_en = 1;
2929
2930         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2931                 txq = dev->data->tx_queues[i];
2932                 txq->tx_ftag_en = tx_ftag_en;
2933                 /* Check if Vector Tx is satisfied */
2934                 if (fm10k_tx_vec_condition_check(txq))
2935                         use_sse = 0;
2936         }
2937
2938         if (use_sse) {
2939                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2940                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2941                         txq = dev->data->tx_queues[i];
2942                         fm10k_txq_vec_setup(txq);
2943                 }
2944                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2945                 dev->tx_pkt_prepare = NULL;
2946         } else {
2947                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2948                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2949                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2950         }
2951 }
2952
2953 static void __attribute__((cold))
2954 fm10k_set_rx_function(struct rte_eth_dev *dev)
2955 {
2956         struct fm10k_dev_info *dev_info =
2957                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2958         uint16_t i, rx_using_sse;
2959         uint16_t rx_ftag_en = 0;
2960
2961         if (fm10k_check_ftag(dev->device->devargs))
2962                 rx_ftag_en = 1;
2963
2964         /* In order to allow Vector Rx there are a few configuration
2965          * conditions to be met.
2966          */
2967         if (!fm10k_rx_vec_condition_check(dev) &&
2968                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2969                 if (dev->data->scattered_rx)
2970                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2971                 else
2972                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2973         } else if (dev->data->scattered_rx)
2974                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2975         else
2976                 dev->rx_pkt_burst = fm10k_recv_pkts;
2977
2978         rx_using_sse =
2979                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2980                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2981
2982         if (rx_using_sse)
2983                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2984         else
2985                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2986
2987         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2988                 return;
2989
2990         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2991                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2992
2993                 rxq->rx_using_sse = rx_using_sse;
2994                 rxq->rx_ftag_en = rx_ftag_en;
2995         }
2996 }
2997
2998 static void
2999 fm10k_params_init(struct rte_eth_dev *dev)
3000 {
3001         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3002         struct fm10k_dev_info *info =
3003                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3004
3005         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3006          * there is no way to get link status without reading BAR4.  Until this
3007          * works, assume we have maximum bandwidth.
3008          * @todo - fix bus info
3009          */
3010         hw->bus_caps.speed = fm10k_bus_speed_8000;
3011         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3012         hw->bus_caps.payload = fm10k_bus_payload_512;
3013         hw->bus.speed = fm10k_bus_speed_8000;
3014         hw->bus.width = fm10k_bus_width_pcie_x8;
3015         hw->bus.payload = fm10k_bus_payload_256;
3016
3017         info->rx_vec_allowed = true;
3018         info->sm_down = false;
3019 }
3020
3021 static int
3022 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3023 {
3024         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3025         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3026         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3027         int diag, i;
3028         struct fm10k_macvlan_filter_info *macvlan;
3029
3030         PMD_INIT_FUNC_TRACE();
3031
3032         dev->dev_ops = &fm10k_eth_dev_ops;
3033         dev->rx_pkt_burst = &fm10k_recv_pkts;
3034         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3035         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3036
3037         /*
3038          * Primary process does the whole initialization, for secondary
3039          * processes, we just select the same Rx and Tx function as primary.
3040          */
3041         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3042                 fm10k_set_rx_function(dev);
3043                 fm10k_set_tx_function(dev);
3044                 return 0;
3045         }
3046
3047         rte_eth_copy_pci_info(dev, pdev);
3048
3049         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3050         memset(macvlan, 0, sizeof(*macvlan));
3051         /* Vendor and Device ID need to be set before init of shared code */
3052         memset(hw, 0, sizeof(*hw));
3053         hw->device_id = pdev->id.device_id;
3054         hw->vendor_id = pdev->id.vendor_id;
3055         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3056         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3057         hw->revision_id = 0;
3058         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3059         if (hw->hw_addr == NULL) {
3060                 PMD_INIT_LOG(ERR, "Bad mem resource."
3061                         " Try to blacklist unused devices.");
3062                 return -EIO;
3063         }
3064
3065         /* Store fm10k_adapter pointer */
3066         hw->back = dev->data->dev_private;
3067
3068         /* Initialize the shared code */
3069         diag = fm10k_init_shared_code(hw);
3070         if (diag != FM10K_SUCCESS) {
3071                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3072                 return -EIO;
3073         }
3074
3075         /* Initialize parameters */
3076         fm10k_params_init(dev);
3077
3078         /* Initialize the hw */
3079         diag = fm10k_init_hw(hw);
3080         if (diag != FM10K_SUCCESS) {
3081                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3082                 return -EIO;
3083         }
3084
3085         /* Initialize MAC address(es) */
3086         dev->data->mac_addrs = rte_zmalloc("fm10k",
3087                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3088         if (dev->data->mac_addrs == NULL) {
3089                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3090                 return -ENOMEM;
3091         }
3092
3093         diag = fm10k_read_mac_addr(hw);
3094
3095         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3096                         &dev->data->mac_addrs[0]);
3097
3098         if (diag != FM10K_SUCCESS ||
3099                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3100
3101                 /* Generate a random addr */
3102                 rte_eth_random_addr(hw->mac.addr);
3103                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3104                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3105                 &dev->data->mac_addrs[0]);
3106         }
3107
3108         /* Reset the hw statistics */
3109         fm10k_stats_reset(dev);
3110
3111         /* Reset the hw */
3112         diag = fm10k_reset_hw(hw);
3113         if (diag != FM10K_SUCCESS) {
3114                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3115                 return -EIO;
3116         }
3117
3118         /* Setup mailbox service */
3119         diag = fm10k_setup_mbx_service(hw);
3120         if (diag != FM10K_SUCCESS) {
3121                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3122                 return -EIO;
3123         }
3124
3125         /*PF/VF has different interrupt handling mechanism */
3126         if (hw->mac.type == fm10k_mac_pf) {
3127                 /* register callback func to eal lib */
3128                 rte_intr_callback_register(intr_handle,
3129                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3130
3131                 /* enable MISC interrupt */
3132                 fm10k_dev_enable_intr_pf(dev);
3133         } else { /* VF */
3134                 rte_intr_callback_register(intr_handle,
3135                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3136
3137                 fm10k_dev_enable_intr_vf(dev);
3138         }
3139
3140         /* Enable intr after callback registered */
3141         rte_intr_enable(intr_handle);
3142
3143         hw->mac.ops.update_int_moderator(hw);
3144
3145         /* Make sure Switch Manager is ready before going forward. */
3146         if (hw->mac.type == fm10k_mac_pf) {
3147                 int switch_ready = 0;
3148
3149                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3150                         fm10k_mbx_lock(hw);
3151                         hw->mac.ops.get_host_state(hw, &switch_ready);
3152                         fm10k_mbx_unlock(hw);
3153                         if (switch_ready)
3154                                 break;
3155                         /* Delay some time to acquire async LPORT_MAP info. */
3156                         rte_delay_us(WAIT_SWITCH_MSG_US);
3157                 }
3158
3159                 if (switch_ready == 0) {
3160                         PMD_INIT_LOG(ERR, "switch is not ready");
3161                         return -1;
3162                 }
3163         }
3164
3165         /*
3166          * Below function will trigger operations on mailbox, acquire lock to
3167          * avoid race condition from interrupt handler. Operations on mailbox
3168          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3169          * will handle and generate an interrupt to our side. Then,  FIFO in
3170          * mailbox will be touched.
3171          */
3172         fm10k_mbx_lock(hw);
3173         /* Enable port first */
3174         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3175                                         MAX_LPORT_NUM, 1);
3176
3177         /* Set unicast mode by default. App can change to other mode in other
3178          * API func.
3179          */
3180         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3181                                         FM10K_XCAST_MODE_NONE);
3182
3183         fm10k_mbx_unlock(hw);
3184
3185         /* Make sure default VID is ready before going forward. */
3186         if (hw->mac.type == fm10k_mac_pf) {
3187                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3188                         if (hw->mac.default_vid)
3189                                 break;
3190                         /* Delay some time to acquire async port VLAN info. */
3191                         rte_delay_us(WAIT_SWITCH_MSG_US);
3192                 }
3193
3194                 if (!hw->mac.default_vid) {
3195                         PMD_INIT_LOG(ERR, "default VID is not ready");
3196                         return -1;
3197                 }
3198         }
3199
3200         /* Add default mac address */
3201         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3202                 MAIN_VSI_POOL_NUMBER);
3203
3204         return 0;
3205 }
3206
3207 static int
3208 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3209 {
3210         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3211         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3212         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3213         PMD_INIT_FUNC_TRACE();
3214
3215         /* only uninitialize in the primary process */
3216         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3217                 return 0;
3218
3219         /* safe to close dev here */
3220         fm10k_dev_close(dev);
3221
3222         dev->dev_ops = NULL;
3223         dev->rx_pkt_burst = NULL;
3224         dev->tx_pkt_burst = NULL;
3225
3226         /* disable uio/vfio intr */
3227         rte_intr_disable(intr_handle);
3228
3229         /*PF/VF has different interrupt handling mechanism */
3230         if (hw->mac.type == fm10k_mac_pf) {
3231                 /* disable interrupt */
3232                 fm10k_dev_disable_intr_pf(dev);
3233
3234                 /* unregister callback func to eal lib */
3235                 rte_intr_callback_unregister(intr_handle,
3236                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3237         } else {
3238                 /* disable interrupt */
3239                 fm10k_dev_disable_intr_vf(dev);
3240
3241                 rte_intr_callback_unregister(intr_handle,
3242                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3243         }
3244
3245         return 0;
3246 }
3247
3248 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3249         struct rte_pci_device *pci_dev)
3250 {
3251         return rte_eth_dev_pci_generic_probe(pci_dev,
3252                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3253 }
3254
3255 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3256 {
3257         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3258 }
3259
3260 /*
3261  * The set of PCI devices this driver supports. This driver will enable both PF
3262  * and SRIOV-VF devices.
3263  */
3264 static const struct rte_pci_id pci_id_fm10k_map[] = {
3265         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3266         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3267         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3268         { .vendor_id = 0, /* sentinel */ },
3269 };
3270
3271 static struct rte_pci_driver rte_pmd_fm10k = {
3272         .id_table = pci_id_fm10k_map,
3273         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3274         .probe = eth_fm10k_pci_probe,
3275         .remove = eth_fm10k_pci_remove,
3276 };
3277
3278 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3279 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3280 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3281
3282 RTE_INIT(fm10k_init_log)
3283 {
3284         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3285         if (fm10k_logtype_init >= 0)
3286                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3287         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3288         if (fm10k_logtype_driver >= 0)
3289                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3290 }