drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 __rte_weak int
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 __rte_weak uint16_t
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 __rte_weak uint16_t
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 __rte_weak int
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 __rte_weak void
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 __rte_weak void
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 __rte_weak int
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 __rte_weak uint16_t
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static int
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return 0;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS) {
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930                 return -EAGAIN;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int
 937 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 938 {
 939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         uint8_t mode;
 941         int status;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Return if it didn't acquire valid glort range */
 946         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 947                 return 0;
 948
 949         if (dev->data->all_multicast == 1)
 950                 mode = FM10K_XCAST_MODE_ALLMULTI;
 951         else
 952                 mode = FM10K_XCAST_MODE_NONE;
 953
 954         fm10k_mbx_lock(hw);
 955         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 956                                 mode);
 957         fm10k_mbx_unlock(hw);
 958
 959         if (status != FM10K_SUCCESS) {
 960                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 961                 return -EAGAIN;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static int
 968 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971         int status;
 972
 973         PMD_INIT_FUNC_TRACE();
 974
 975         /* Return if it didn't acquire valid glort range */
 976         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 977                 return 0;
 978
 979         /* If promiscuous mode is enabled, it doesn't make sense to enable
 980          * allmulticast and disable promiscuous since fm10k only can select
 981          * one of the modes.
 982          */
 983         if (dev->data->promiscuous) {
 984                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 985                         "needn't enable allmulticast");
 986                 return 0;
 987         }
 988
 989         fm10k_mbx_lock(hw);
 990         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 991                                 FM10K_XCAST_MODE_ALLMULTI);
 992         fm10k_mbx_unlock(hw);
 993
 994         if (status != FM10K_SUCCESS) {
 995                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 996                 return -EAGAIN;
 997         }
 998
 999         return 0;
1000 }
1001
1002 static int
1003 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1004 {
1005         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1006         int status;
1007
1008         PMD_INIT_FUNC_TRACE();
1009
1010         /* Return if it didn't acquire valid glort range */
1011         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1012                 return 0;
1013
1014         if (dev->data->promiscuous) {
1015                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1016                         "since promisc mode is enabled");
1017                 return -EINVAL;
1018         }
1019
1020         fm10k_mbx_lock(hw);
1021         /* Change mode to unicast mode */
1022         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1023                                 FM10K_XCAST_MODE_NONE);
1024         fm10k_mbx_unlock(hw);
1025
1026         if (status != FM10K_SUCCESS) {
1027                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1028                 return -EAGAIN;
1029         }
1030
1031         return 0;
1032 }
1033
1034 static void
1035 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1036 {
1037         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1038         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1039         uint16_t nb_queue_pools;
1040         struct fm10k_macvlan_filter_info *macvlan;
1041
1042         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1043         nb_queue_pools = macvlan->nb_queue_pools;
1044         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1045         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1046
1047         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1048         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1049         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1050                         hw->mac.dglort_map;
1051         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1052         /* Configure VMDQ/RSS DGlort Decoder */
1053         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1054
1055         /* Flow Director configurations, only queue number is valid. */
1056         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1057         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1058                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1059         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1060         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1061
1062         /* Invalidate all other GLORT entries */
1063         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1064                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1065                                 FM10K_DGLORTMAP_NONE);
1066 }
1067
1068 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1069 static int
1070 fm10k_dev_start(struct rte_eth_dev *dev)
1071 {
1072         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1073         int i, diag;
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         /* stop, init, then start the hw */
1078         diag = fm10k_stop_hw(hw);
1079         if (diag != FM10K_SUCCESS) {
1080                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1081                 return -EIO;
1082         }
1083
1084         diag = fm10k_init_hw(hw);
1085         if (diag != FM10K_SUCCESS) {
1086                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1087                 return -EIO;
1088         }
1089
1090         diag = fm10k_start_hw(hw);
1091         if (diag != FM10K_SUCCESS) {
1092                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1093                 return -EIO;
1094         }
1095
1096         diag = fm10k_dev_tx_init(dev);
1097         if (diag) {
1098                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1099                 return diag;
1100         }
1101
1102         if (fm10k_dev_rxq_interrupt_setup(dev))
1103                 return -EIO;
1104
1105         diag = fm10k_dev_rx_init(dev);
1106         if (diag) {
1107                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1108                 return diag;
1109         }
1110
1111         if (hw->mac.type == fm10k_mac_pf)
1112                 fm10k_dev_dglort_map_configure(dev);
1113
1114         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1115                 struct fm10k_rx_queue *rxq;
1116                 rxq = dev->data->rx_queues[i];
1117
1118                 if (rxq->rx_deferred_start)
1119                         continue;
1120                 diag = fm10k_dev_rx_queue_start(dev, i);
1121                 if (diag != 0) {
1122                         int j;
1123                         for (j = 0; j < i; ++j)
1124                                 rx_queue_clean(dev->data->rx_queues[j]);
1125                         return diag;
1126                 }
1127         }
1128
1129         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1130                 struct fm10k_tx_queue *txq;
1131                 txq = dev->data->tx_queues[i];
1132
1133                 if (txq->tx_deferred_start)
1134                         continue;
1135                 diag = fm10k_dev_tx_queue_start(dev, i);
1136                 if (diag != 0) {
1137                         int j;
1138                         for (j = 0; j < i; ++j)
1139                                 tx_queue_clean(dev->data->tx_queues[j]);
1140                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1141                                 rx_queue_clean(dev->data->rx_queues[j]);
1142                         return diag;
1143                 }
1144         }
1145
1146         /* Update default vlan when not in VMDQ mode */
1147         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1148                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1149
1150         fm10k_link_update(dev, 0);
1151
1152         return 0;
1153 }
1154
1155 static void
1156 fm10k_dev_stop(struct rte_eth_dev *dev)
1157 {
1158         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1159         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1160         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1161         int i;
1162
1163         PMD_INIT_FUNC_TRACE();
1164
1165         if (dev->data->tx_queues)
1166                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1167                         fm10k_dev_tx_queue_stop(dev, i);
1168
1169         if (dev->data->rx_queues)
1170                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1171                         fm10k_dev_rx_queue_stop(dev, i);
1172
1173         /* Disable datapath event */
1174         if (rte_intr_dp_is_en(intr_handle)) {
1175                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1176                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1177                                 3 << FM10K_RXINT_TIMER_SHIFT);
1178                         if (hw->mac.type == fm10k_mac_pf)
1179                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1180                                         FM10K_ITR_MASK_SET);
1181                         else
1182                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1183                                         FM10K_ITR_MASK_SET);
1184                 }
1185         }
1186         /* Clean datapath event and queue/vec mapping */
1187         rte_intr_efd_disable(intr_handle);
1188         rte_free(intr_handle->intr_vec);
1189         intr_handle->intr_vec = NULL;
1190 }
1191
1192 static void
1193 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1194 {
1195         int i;
1196
1197         PMD_INIT_FUNC_TRACE();
1198
1199         if (dev->data->tx_queues) {
1200                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1201                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1202
1203                         tx_queue_free(txq);
1204                 }
1205         }
1206
1207         if (dev->data->rx_queues) {
1208                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1209                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1210         }
1211 }
1212
1213 static void
1214 fm10k_dev_close(struct rte_eth_dev *dev)
1215 {
1216         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1217
1218         PMD_INIT_FUNC_TRACE();
1219
1220         fm10k_mbx_lock(hw);
1221         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1222                 MAX_LPORT_NUM, false);
1223         fm10k_mbx_unlock(hw);
1224
1225         /* allow 100ms for device to quiesce */
1226         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1227
1228         /* Stop mailbox service first */
1229         fm10k_close_mbx_service(hw);
1230         fm10k_dev_stop(dev);
1231         fm10k_dev_queue_release(dev);
1232         fm10k_stop_hw(hw);
1233 }
1234
1235 static int
1236 fm10k_link_update(struct rte_eth_dev *dev,
1237         __rte_unused int wait_to_complete)
1238 {
1239         struct fm10k_dev_info *dev_info =
1240                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1241         PMD_INIT_FUNC_TRACE();
1242
1243         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1244         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1245         dev->data->dev_link.link_status =
1246                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1247         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1248
1249         return 0;
1250 }
1251
1252 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1253         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1254 {
1255         unsigned i, q;
1256         unsigned count = 0;
1257
1258         if (xstats_names != NULL) {
1259                 /* Note: limit checked in rte_eth_xstats_names() */
1260
1261                 /* Global stats */
1262                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1263                         snprintf(xstats_names[count].name,
1264                                 sizeof(xstats_names[count].name),
1265                                 "%s", fm10k_hw_stats_strings[count].name);
1266                         count++;
1267                 }
1268
1269                 /* PF queue stats */
1270                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1271                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1272                                 snprintf(xstats_names[count].name,
1273                                         sizeof(xstats_names[count].name),
1274                                         "rx_q%u_%s", q,
1275                                         fm10k_hw_stats_rx_q_strings[i].name);
1276                                 count++;
1277                         }
1278                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1279                                 snprintf(xstats_names[count].name,
1280                                         sizeof(xstats_names[count].name),
1281                                         "tx_q%u_%s", q,
1282                                         fm10k_hw_stats_tx_q_strings[i].name);
1283                                 count++;
1284                         }
1285                 }
1286         }
1287         return FM10K_NB_XSTATS;
1288 }
1289
1290 static int
1291 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1292                  unsigned n)
1293 {
1294         struct fm10k_hw_stats *hw_stats =
1295                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1296         unsigned i, q, count = 0;
1297
1298         if (n < FM10K_NB_XSTATS)
1299                 return FM10K_NB_XSTATS;
1300
1301         /* Global stats */
1302         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1303                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1304                         fm10k_hw_stats_strings[count].offset);
1305                 xstats[count].id = count;
1306                 count++;
1307         }
1308
1309         /* PF queue stats */
1310         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1311                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1312                         xstats[count].value =
1313                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1314                                 fm10k_hw_stats_rx_q_strings[i].offset);
1315                         xstats[count].id = count;
1316                         count++;
1317                 }
1318                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1319                         xstats[count].value =
1320                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1321                                 fm10k_hw_stats_tx_q_strings[i].offset);
1322                         xstats[count].id = count;
1323                         count++;
1324                 }
1325         }
1326
1327         return FM10K_NB_XSTATS;
1328 }
1329
1330 static int
1331 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1332 {
1333         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1334         struct fm10k_hw *hw =
1335                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1336         struct fm10k_hw_stats *hw_stats =
1337                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1338         int i;
1339
1340         PMD_INIT_FUNC_TRACE();
1341
1342         fm10k_update_hw_stats(hw, hw_stats);
1343
1344         ipackets = opackets = ibytes = obytes = imissed = 0;
1345         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1346                 (i < hw->mac.max_queues); ++i) {
1347                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1348                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1349                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1350                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1351                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1352                 ipackets += stats->q_ipackets[i];
1353                 opackets += stats->q_opackets[i];
1354                 ibytes   += stats->q_ibytes[i];
1355                 obytes   += stats->q_obytes[i];
1356                 imissed  += stats->q_errors[i];
1357         }
1358         stats->ipackets = ipackets;
1359         stats->opackets = opackets;
1360         stats->ibytes = ibytes;
1361         stats->obytes = obytes;
1362         stats->imissed = imissed;
1363         return 0;
1364 }
1365
1366 static int
1367 fm10k_stats_reset(struct rte_eth_dev *dev)
1368 {
1369         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1370         struct fm10k_hw_stats *hw_stats =
1371                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1372
1373         PMD_INIT_FUNC_TRACE();
1374
1375         memset(hw_stats, 0, sizeof(*hw_stats));
1376         fm10k_rebind_hw_stats(hw, hw_stats);
1377
1378         return 0;
1379 }
1380
1381 static int
1382 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1383         struct rte_eth_dev_info *dev_info)
1384 {
1385         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1386         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1387
1388         PMD_INIT_FUNC_TRACE();
1389
1390         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1391         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1392         dev_info->max_rx_queues      = hw->mac.max_queues;
1393         dev_info->max_tx_queues      = hw->mac.max_queues;
1394         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1395         dev_info->max_hash_mac_addrs = 0;
1396         dev_info->max_vfs            = pdev->max_vfs;
1397         dev_info->vmdq_pool_base     = 0;
1398         dev_info->vmdq_queue_base    = 0;
1399         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1400         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1401         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1402         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1403                                     dev_info->rx_queue_offload_capa;
1404         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1405         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1406                                     dev_info->tx_queue_offload_capa;
1407
1408         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1409         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1410         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1411                                         ETH_RSS_IPV6 |
1412                                         ETH_RSS_IPV6_EX |
1413                                         ETH_RSS_NONFRAG_IPV4_TCP |
1414                                         ETH_RSS_NONFRAG_IPV6_TCP |
1415                                         ETH_RSS_IPV6_TCP_EX |
1416                                         ETH_RSS_NONFRAG_IPV4_UDP |
1417                                         ETH_RSS_NONFRAG_IPV6_UDP |
1418                                         ETH_RSS_IPV6_UDP_EX;
1419
1420         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1421                 .rx_thresh = {
1422                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1423                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1424                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1425                 },
1426                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1427                 .rx_drop_en = 0,
1428                 .offloads = 0,
1429         };
1430
1431         dev_info->default_txconf = (struct rte_eth_txconf) {
1432                 .tx_thresh = {
1433                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1434                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1435                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1436                 },
1437                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1438                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1439                 .offloads = 0,
1440         };
1441
1442         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1443                 .nb_max = FM10K_MAX_RX_DESC,
1444                 .nb_min = FM10K_MIN_RX_DESC,
1445                 .nb_align = FM10K_MULT_RX_DESC,
1446         };
1447
1448         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1449                 .nb_max = FM10K_MAX_TX_DESC,
1450                 .nb_min = FM10K_MIN_TX_DESC,
1451                 .nb_align = FM10K_MULT_TX_DESC,
1452                 .nb_seg_max = FM10K_TX_MAX_SEG,
1453                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1454         };
1455
1456         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1457                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1458                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1459
1460         return 0;
1461 }
1462
1463 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1464 static const uint32_t *
1465 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1466 {
1467         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1468             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1469                 static uint32_t ptypes[] = {
1470                         /* refers to rx_desc_to_ol_flags() */
1471                         RTE_PTYPE_L2_ETHER,
1472                         RTE_PTYPE_L3_IPV4,
1473                         RTE_PTYPE_L3_IPV4_EXT,
1474                         RTE_PTYPE_L3_IPV6,
1475                         RTE_PTYPE_L3_IPV6_EXT,
1476                         RTE_PTYPE_L4_TCP,
1477                         RTE_PTYPE_L4_UDP,
1478                         RTE_PTYPE_UNKNOWN
1479                 };
1480
1481                 return ptypes;
1482         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1483                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1484                 static uint32_t ptypes_vec[] = {
1485                         /* refers to fm10k_desc_to_pktype_v() */
1486                         RTE_PTYPE_L3_IPV4,
1487                         RTE_PTYPE_L3_IPV4_EXT,
1488                         RTE_PTYPE_L3_IPV6,
1489                         RTE_PTYPE_L3_IPV6_EXT,
1490                         RTE_PTYPE_L4_TCP,
1491                         RTE_PTYPE_L4_UDP,
1492                         RTE_PTYPE_TUNNEL_GENEVE,
1493                         RTE_PTYPE_TUNNEL_NVGRE,
1494                         RTE_PTYPE_TUNNEL_VXLAN,
1495                         RTE_PTYPE_TUNNEL_GRE,
1496                         RTE_PTYPE_UNKNOWN
1497                 };
1498
1499                 return ptypes_vec;
1500         }
1501
1502         return NULL;
1503 }
1504 #else
1505 static const uint32_t *
1506 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1507 {
1508         return NULL;
1509 }
1510 #endif
1511
1512 static int
1513 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1514 {
1515         s32 result;
1516         uint16_t mac_num = 0;
1517         uint32_t vid_idx, vid_bit, mac_index;
1518         struct fm10k_hw *hw;
1519         struct fm10k_macvlan_filter_info *macvlan;
1520         struct rte_eth_dev_data *data = dev->data;
1521
1522         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1523         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1524
1525         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1526                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1527                 return -EINVAL;
1528         }
1529
1530         if (vlan_id > ETH_VLAN_ID_MAX) {
1531                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1532                 return -EINVAL;
1533         }
1534
1535         vid_idx = FM10K_VFTA_IDX(vlan_id);
1536         vid_bit = FM10K_VFTA_BIT(vlan_id);
1537         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1538         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1539                 return 0;
1540         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1541         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1542                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1543                         "in the VLAN filter table");
1544                 return -EINVAL;
1545         }
1546
1547         fm10k_mbx_lock(hw);
1548         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1549         fm10k_mbx_unlock(hw);
1550         if (result != FM10K_SUCCESS) {
1551                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1552                 return -EIO;
1553         }
1554
1555         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1556                         (result == FM10K_SUCCESS); mac_index++) {
1557                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1558                         continue;
1559                 if (mac_num > macvlan->mac_num - 1) {
1560                         PMD_INIT_LOG(ERR, "MAC address number "
1561                                         "not match");
1562                         break;
1563                 }
1564                 fm10k_mbx_lock(hw);
1565                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1566                         data->mac_addrs[mac_index].addr_bytes,
1567                         vlan_id, on, 0);
1568                 fm10k_mbx_unlock(hw);
1569                 mac_num++;
1570         }
1571         if (result != FM10K_SUCCESS) {
1572                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1573                 return -EIO;
1574         }
1575
1576         if (on) {
1577                 macvlan->vlan_num++;
1578                 macvlan->vfta[vid_idx] |= vid_bit;
1579         } else {
1580                 macvlan->vlan_num--;
1581                 macvlan->vfta[vid_idx] &= ~vid_bit;
1582         }
1583         return 0;
1584 }
1585
1586 static int
1587 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1588 {
1589         if (mask & ETH_VLAN_STRIP_MASK) {
1590                 if (!(dev->data->dev_conf.rxmode.offloads &
1591                         DEV_RX_OFFLOAD_VLAN_STRIP))
1592                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1593                                         "always on in fm10k");
1594         }
1595
1596         if (mask & ETH_VLAN_EXTEND_MASK) {
1597                 if (dev->data->dev_conf.rxmode.offloads &
1598                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1599                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1600                                         "supported in fm10k");
1601         }
1602
1603         if (mask & ETH_VLAN_FILTER_MASK) {
1604                 if (!(dev->data->dev_conf.rxmode.offloads &
1605                         DEV_RX_OFFLOAD_VLAN_FILTER))
1606                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1607         }
1608
1609         return 0;
1610 }
1611
1612 /* Add/Remove a MAC address, and update filters to main VSI */
1613 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1614                 const u8 *mac, bool add, uint32_t pool)
1615 {
1616         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1617         struct fm10k_macvlan_filter_info *macvlan;
1618         uint32_t i, j, k;
1619
1620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1621
1622         if (pool != MAIN_VSI_POOL_NUMBER) {
1623                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1624                         "mac to pool %u", pool);
1625                 return;
1626         }
1627         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1628                 if (!macvlan->vfta[j])
1629                         continue;
1630                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1631                         if (!(macvlan->vfta[j] & (1 << k)))
1632                                 continue;
1633                         if (i + 1 > macvlan->vlan_num) {
1634                                 PMD_INIT_LOG(ERR, "vlan number not match");
1635                                 return;
1636                         }
1637                         fm10k_mbx_lock(hw);
1638                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1639                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1640                         fm10k_mbx_unlock(hw);
1641                         i++;
1642                 }
1643         }
1644 }
1645
1646 /* Add/Remove a MAC address, and update filters to VMDQ */
1647 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1648                 const u8 *mac, bool add, uint32_t pool)
1649 {
1650         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1651         struct fm10k_macvlan_filter_info *macvlan;
1652         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1653         uint32_t i;
1654
1655         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1656         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1657
1658         if (pool > macvlan->nb_queue_pools) {
1659                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1660                         " Max pool is %u",
1661                         pool, macvlan->nb_queue_pools);
1662                 return;
1663         }
1664         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1665                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1666                         continue;
1667                 fm10k_mbx_lock(hw);
1668                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1669                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1670                 fm10k_mbx_unlock(hw);
1671         }
1672 }
1673
1674 /* Add/Remove a MAC address, and update filters */
1675 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1676                 const u8 *mac, bool add, uint32_t pool)
1677 {
1678         struct fm10k_macvlan_filter_info *macvlan;
1679
1680         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1681
1682         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1683                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1684         else
1685                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1686
1687         if (add)
1688                 macvlan->mac_num++;
1689         else
1690                 macvlan->mac_num--;
1691 }
1692
1693 /* Add a MAC address, and update filters */
1694 static int
1695 fm10k_macaddr_add(struct rte_eth_dev *dev,
1696                 struct rte_ether_addr *mac_addr,
1697                 uint32_t index,
1698                 uint32_t pool)
1699 {
1700         struct fm10k_macvlan_filter_info *macvlan;
1701
1702         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1703         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1704         macvlan->mac_vmdq_id[index] = pool;
1705         return 0;
1706 }
1707
1708 /* Remove a MAC address, and update filters */
1709 static void
1710 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1711 {
1712         struct rte_eth_dev_data *data = dev->data;
1713         struct fm10k_macvlan_filter_info *macvlan;
1714
1715         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1716         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1717                         FALSE, macvlan->mac_vmdq_id[index]);
1718         macvlan->mac_vmdq_id[index] = 0;
1719 }
1720
1721 static inline int
1722 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1723 {
1724         if ((request < min) || (request > max) || ((request % mult) != 0))
1725                 return -1;
1726         else
1727                 return 0;
1728 }
1729
1730
1731 static inline int
1732 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1733 {
1734         if ((request < min) || (request > max) || ((div % request) != 0))
1735                 return -1;
1736         else
1737                 return 0;
1738 }
1739
1740 static inline int
1741 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1742 {
1743         uint16_t rx_free_thresh;
1744
1745         if (conf->rx_free_thresh == 0)
1746                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1747         else
1748                 rx_free_thresh = conf->rx_free_thresh;
1749
1750         /* make sure the requested threshold satisfies the constraints */
1751         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1752                         FM10K_RX_FREE_THRESH_MAX(q),
1753                         FM10K_RX_FREE_THRESH_DIV(q),
1754                         rx_free_thresh)) {
1755                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1756                         "less than or equal to %u, "
1757                         "greater than or equal to %u, "
1758                         "and a divisor of %u",
1759                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1760                         FM10K_RX_FREE_THRESH_MIN(q),
1761                         FM10K_RX_FREE_THRESH_DIV(q));
1762                 return -EINVAL;
1763         }
1764
1765         q->alloc_thresh = rx_free_thresh;
1766         q->drop_en = conf->rx_drop_en;
1767         q->rx_deferred_start = conf->rx_deferred_start;
1768
1769         return 0;
1770 }
1771
1772 /*
1773  * Hardware requires specific alignment for Rx packet buffers. At
1774  * least one of the following two conditions must be satisfied.
1775  *  1. Address is 512B aligned
1776  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1777  *
1778  * As such, the driver may need to adjust the DMA address within the
1779  * buffer by up to 512B.
1780  *
1781  * return 1 if the element size is valid, otherwise return 0.
1782  */
1783 static int
1784 mempool_element_size_valid(struct rte_mempool *mp)
1785 {
1786         uint32_t min_size;
1787
1788         /* elt_size includes mbuf header and headroom */
1789         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1790                         RTE_PKTMBUF_HEADROOM;
1791
1792         /* account for up to 512B of alignment */
1793         min_size -= FM10K_RX_DATABUF_ALIGN;
1794
1795         /* sanity check for overflow */
1796         if (min_size > mp->elt_size)
1797                 return 0;
1798
1799         /* size is valid */
1800         return 1;
1801 }
1802
1803 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1804 {
1805         RTE_SET_USED(dev);
1806
1807         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1808 }
1809
1810 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1811 {
1812         RTE_SET_USED(dev);
1813
1814         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1815                            DEV_RX_OFFLOAD_VLAN_FILTER |
1816                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1817                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1818                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1819                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1820                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1821 }
1822
1823 static int
1824 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1825         uint16_t nb_desc, unsigned int socket_id,
1826         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1827 {
1828         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1829         struct fm10k_dev_info *dev_info =
1830                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1831         struct fm10k_rx_queue *q;
1832         const struct rte_memzone *mz;
1833         uint64_t offloads;
1834
1835         PMD_INIT_FUNC_TRACE();
1836
1837         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1838
1839         /* make sure the mempool element size can account for alignment. */
1840         if (!mempool_element_size_valid(mp)) {
1841                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1842                 return -EINVAL;
1843         }
1844
1845         /* make sure a valid number of descriptors have been requested */
1846         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1847                                 FM10K_MULT_RX_DESC, nb_desc)) {
1848                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1849                         "less than or equal to %"PRIu32", "
1850                         "greater than or equal to %u, "
1851                         "and a multiple of %u",
1852                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1853                         FM10K_MULT_RX_DESC);
1854                 return -EINVAL;
1855         }
1856
1857         /*
1858          * if this queue existed already, free the associated memory. The
1859          * queue cannot be reused in case we need to allocate memory on
1860          * different socket than was previously used.
1861          */
1862         if (dev->data->rx_queues[queue_id] != NULL) {
1863                 rx_queue_free(dev->data->rx_queues[queue_id]);
1864                 dev->data->rx_queues[queue_id] = NULL;
1865         }
1866
1867         /* allocate memory for the queue structure */
1868         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1869                                 socket_id);
1870         if (q == NULL) {
1871                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1872                 return -ENOMEM;
1873         }
1874
1875         /* setup queue */
1876         q->mp = mp;
1877         q->nb_desc = nb_desc;
1878         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1879         q->port_id = dev->data->port_id;
1880         q->queue_id = queue_id;
1881         q->tail_ptr = (volatile uint32_t *)
1882                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1883         q->offloads = offloads;
1884         if (handle_rxconf(q, conf))
1885                 return -EINVAL;
1886
1887         /* allocate memory for the software ring */
1888         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1889                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1890                         RTE_CACHE_LINE_SIZE, socket_id);
1891         if (q->sw_ring == NULL) {
1892                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1893                 rte_free(q);
1894                 return -ENOMEM;
1895         }
1896
1897         /*
1898          * allocate memory for the hardware descriptor ring. A memzone large
1899          * enough to hold the maximum ring size is requested to allow for
1900          * resizing in later calls to the queue setup function.
1901          */
1902         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1903                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1904                                       socket_id);
1905         if (mz == NULL) {
1906                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1907                 rte_free(q->sw_ring);
1908                 rte_free(q);
1909                 return -ENOMEM;
1910         }
1911         q->hw_ring = mz->addr;
1912         q->hw_ring_phys_addr = mz->iova;
1913
1914         /* Check if number of descs satisfied Vector requirement */
1915         if (!rte_is_power_of_2(nb_desc)) {
1916                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1917                                     "preconditions - canceling the feature for "
1918                                     "the whole port[%d]",
1919                              q->queue_id, q->port_id);
1920                 dev_info->rx_vec_allowed = false;
1921         } else
1922                 fm10k_rxq_vec_setup(q);
1923
1924         dev->data->rx_queues[queue_id] = q;
1925         return 0;
1926 }
1927
1928 static void
1929 fm10k_rx_queue_release(void *queue)
1930 {
1931         PMD_INIT_FUNC_TRACE();
1932
1933         rx_queue_free(queue);
1934 }
1935
1936 static inline int
1937 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1938 {
1939         uint16_t tx_free_thresh;
1940         uint16_t tx_rs_thresh;
1941
1942         /* constraint MACROs require that tx_free_thresh is configured
1943          * before tx_rs_thresh */
1944         if (conf->tx_free_thresh == 0)
1945                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1946         else
1947                 tx_free_thresh = conf->tx_free_thresh;
1948
1949         /* make sure the requested threshold satisfies the constraints */
1950         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1951                         FM10K_TX_FREE_THRESH_MAX(q),
1952                         FM10K_TX_FREE_THRESH_DIV(q),
1953                         tx_free_thresh)) {
1954                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1955                         "less than or equal to %u, "
1956                         "greater than or equal to %u, "
1957                         "and a divisor of %u",
1958                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1959                         FM10K_TX_FREE_THRESH_MIN(q),
1960                         FM10K_TX_FREE_THRESH_DIV(q));
1961                 return -EINVAL;
1962         }
1963
1964         q->free_thresh = tx_free_thresh;
1965
1966         if (conf->tx_rs_thresh == 0)
1967                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1968         else
1969                 tx_rs_thresh = conf->tx_rs_thresh;
1970
1971         q->tx_deferred_start = conf->tx_deferred_start;
1972
1973         /* make sure the requested threshold satisfies the constraints */
1974         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1975                         FM10K_TX_RS_THRESH_MAX(q),
1976                         FM10K_TX_RS_THRESH_DIV(q),
1977                         tx_rs_thresh)) {
1978                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1979                         "less than or equal to %u, "
1980                         "greater than or equal to %u, "
1981                         "and a divisor of %u",
1982                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1983                         FM10K_TX_RS_THRESH_MIN(q),
1984                         FM10K_TX_RS_THRESH_DIV(q));
1985                 return -EINVAL;
1986         }
1987
1988         q->rs_thresh = tx_rs_thresh;
1989
1990         return 0;
1991 }
1992
1993 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1994 {
1995         RTE_SET_USED(dev);
1996
1997         return 0;
1998 }
1999
2000 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
2001 {
2002         RTE_SET_USED(dev);
2003
2004         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
2005                           DEV_TX_OFFLOAD_MULTI_SEGS  |
2006                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
2007                           DEV_TX_OFFLOAD_UDP_CKSUM   |
2008                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2009                           DEV_TX_OFFLOAD_TCP_TSO);
2010 }
2011
2012 static int
2013 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2014         uint16_t nb_desc, unsigned int socket_id,
2015         const struct rte_eth_txconf *conf)
2016 {
2017         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2018         struct fm10k_tx_queue *q;
2019         const struct rte_memzone *mz;
2020         uint64_t offloads;
2021
2022         PMD_INIT_FUNC_TRACE();
2023
2024         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2025
2026         /* make sure a valid number of descriptors have been requested */
2027         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2028                                 FM10K_MULT_TX_DESC, nb_desc)) {
2029                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2030                         "less than or equal to %"PRIu32", "
2031                         "greater than or equal to %u, "
2032                         "and a multiple of %u",
2033                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2034                         FM10K_MULT_TX_DESC);
2035                 return -EINVAL;
2036         }
2037
2038         /*
2039          * if this queue existed already, free the associated memory. The
2040          * queue cannot be reused in case we need to allocate memory on
2041          * different socket than was previously used.
2042          */
2043         if (dev->data->tx_queues[queue_id] != NULL) {
2044                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2045
2046                 tx_queue_free(txq);
2047                 dev->data->tx_queues[queue_id] = NULL;
2048         }
2049
2050         /* allocate memory for the queue structure */
2051         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2052                                 socket_id);
2053         if (q == NULL) {
2054                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2055                 return -ENOMEM;
2056         }
2057
2058         /* setup queue */
2059         q->nb_desc = nb_desc;
2060         q->port_id = dev->data->port_id;
2061         q->queue_id = queue_id;
2062         q->offloads = offloads;
2063         q->ops = &def_txq_ops;
2064         q->tail_ptr = (volatile uint32_t *)
2065                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2066         if (handle_txconf(q, conf))
2067                 return -EINVAL;
2068
2069         /* allocate memory for the software ring */
2070         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2071                                         nb_desc * sizeof(struct rte_mbuf *),
2072                                         RTE_CACHE_LINE_SIZE, socket_id);
2073         if (q->sw_ring == NULL) {
2074                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2075                 rte_free(q);
2076                 return -ENOMEM;
2077         }
2078
2079         /*
2080          * allocate memory for the hardware descriptor ring. A memzone large
2081          * enough to hold the maximum ring size is requested to allow for
2082          * resizing in later calls to the queue setup function.
2083          */
2084         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2085                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2086                                       socket_id);
2087         if (mz == NULL) {
2088                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2089                 rte_free(q->sw_ring);
2090                 rte_free(q);
2091                 return -ENOMEM;
2092         }
2093         q->hw_ring = mz->addr;
2094         q->hw_ring_phys_addr = mz->iova;
2095
2096         /*
2097          * allocate memory for the RS bit tracker. Enough slots to hold the
2098          * descriptor index for each RS bit needing to be set are required.
2099          */
2100         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2101                                 ((nb_desc + 1) / q->rs_thresh) *
2102                                 sizeof(uint16_t),
2103                                 RTE_CACHE_LINE_SIZE, socket_id);
2104         if (q->rs_tracker.list == NULL) {
2105                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2106                 rte_free(q->sw_ring);
2107                 rte_free(q);
2108                 return -ENOMEM;
2109         }
2110
2111         dev->data->tx_queues[queue_id] = q;
2112         return 0;
2113 }
2114
2115 static void
2116 fm10k_tx_queue_release(void *queue)
2117 {
2118         struct fm10k_tx_queue *q = queue;
2119         PMD_INIT_FUNC_TRACE();
2120
2121         tx_queue_free(q);
2122 }
2123
2124 static int
2125 fm10k_reta_update(struct rte_eth_dev *dev,
2126                         struct rte_eth_rss_reta_entry64 *reta_conf,
2127                         uint16_t reta_size)
2128 {
2129         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2130         uint16_t i, j, idx, shift;
2131         uint8_t mask;
2132         uint32_t reta;
2133
2134         PMD_INIT_FUNC_TRACE();
2135
2136         if (reta_size > FM10K_MAX_RSS_INDICES) {
2137                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2138                         "(%d) doesn't match the number hardware can supported "
2139                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2140                 return -EINVAL;
2141         }
2142
2143         /*
2144          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2145          * 128-entries in 32 registers
2146          */
2147         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2148                 idx = i / RTE_RETA_GROUP_SIZE;
2149                 shift = i % RTE_RETA_GROUP_SIZE;
2150                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2151                                 BIT_MASK_PER_UINT32);
2152                 if (mask == 0)
2153                         continue;
2154
2155                 reta = 0;
2156                 if (mask != BIT_MASK_PER_UINT32)
2157                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2158
2159                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2160                         if (mask & (0x1 << j)) {
2161                                 if (mask != 0xF)
2162                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2163                                 reta |= reta_conf[idx].reta[shift + j] <<
2164                                                 (CHAR_BIT * j);
2165                         }
2166                 }
2167                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2168         }
2169
2170         return 0;
2171 }
2172
2173 static int
2174 fm10k_reta_query(struct rte_eth_dev *dev,
2175                         struct rte_eth_rss_reta_entry64 *reta_conf,
2176                         uint16_t reta_size)
2177 {
2178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2179         uint16_t i, j, idx, shift;
2180         uint8_t mask;
2181         uint32_t reta;
2182
2183         PMD_INIT_FUNC_TRACE();
2184
2185         if (reta_size < FM10K_MAX_RSS_INDICES) {
2186                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2187                         "(%d) doesn't match the number hardware can supported "
2188                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2189                 return -EINVAL;
2190         }
2191
2192         /*
2193          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2194          * 128-entries in 32 registers
2195          */
2196         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2197                 idx = i / RTE_RETA_GROUP_SIZE;
2198                 shift = i % RTE_RETA_GROUP_SIZE;
2199                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2200                                 BIT_MASK_PER_UINT32);
2201                 if (mask == 0)
2202                         continue;
2203
2204                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2205                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2206                         if (mask & (0x1 << j))
2207                                 reta_conf[idx].reta[shift + j] = ((reta >>
2208                                         CHAR_BIT * j) & UINT8_MAX);
2209                 }
2210         }
2211
2212         return 0;
2213 }
2214
2215 static int
2216 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2217         struct rte_eth_rss_conf *rss_conf)
2218 {
2219         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2220         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2221         uint32_t mrqc;
2222         uint64_t hf = rss_conf->rss_hf;
2223         int i;
2224
2225         PMD_INIT_FUNC_TRACE();
2226
2227         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2228                                 FM10K_RSSRK_ENTRIES_PER_REG))
2229                 return -EINVAL;
2230
2231         if (hf == 0)
2232                 return -EINVAL;
2233
2234         mrqc = 0;
2235         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2236         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2237         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2238         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2239         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2240         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2241         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2242         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2243         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2244
2245         /* If the mapping doesn't fit any supported, return */
2246         if (mrqc == 0)
2247                 return -EINVAL;
2248
2249         if (key != NULL)
2250                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2251                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2252
2253         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2254
2255         return 0;
2256 }
2257
2258 static int
2259 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2260         struct rte_eth_rss_conf *rss_conf)
2261 {
2262         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2263         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2264         uint32_t mrqc;
2265         uint64_t hf;
2266         int i;
2267
2268         PMD_INIT_FUNC_TRACE();
2269
2270         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2271                                 FM10K_RSSRK_ENTRIES_PER_REG))
2272                 return -EINVAL;
2273
2274         if (key != NULL)
2275                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2276                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2277
2278         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2279         hf = 0;
2280         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2281         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2282         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2283         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2284         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2285         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2286         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2287         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2288         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2289
2290         rss_conf->rss_hf = hf;
2291
2292         return 0;
2293 }
2294
2295 static void
2296 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2297 {
2298         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2299         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2300
2301         /* Bind all local non-queue interrupt to vector 0 */
2302         int_map |= FM10K_MISC_VEC_ID;
2303
2304         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2305         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2306         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2307         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2308         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2309         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2310
2311         /* Enable misc causes */
2312         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2313                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2314                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2315                                 FM10K_EIMR_ENABLE(MAILBOX) |
2316                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2317                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2318                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2319                                 FM10K_EIMR_ENABLE(VFLR));
2320
2321         /* Enable ITR 0 */
2322         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2323                                         FM10K_ITR_MASK_CLEAR);
2324         FM10K_WRITE_FLUSH(hw);
2325 }
2326
2327 static void
2328 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2329 {
2330         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2331         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2332
2333         int_map |= FM10K_MISC_VEC_ID;
2334
2335         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2336         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2337         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2338         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2339         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2340         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2341
2342         /* Disable misc causes */
2343         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2344                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2345                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2346                                 FM10K_EIMR_DISABLE(MAILBOX) |
2347                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2348                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2349                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2350                                 FM10K_EIMR_DISABLE(VFLR));
2351
2352         /* Disable ITR 0 */
2353         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2354         FM10K_WRITE_FLUSH(hw);
2355 }
2356
2357 static void
2358 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2359 {
2360         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2361         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2362
2363         /* Bind all local non-queue interrupt to vector 0 */
2364         int_map |= FM10K_MISC_VEC_ID;
2365
2366         /* Only INT 0 available, other 15 are reserved. */
2367         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2368
2369         /* Enable ITR 0 */
2370         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2371                                         FM10K_ITR_MASK_CLEAR);
2372         FM10K_WRITE_FLUSH(hw);
2373 }
2374
2375 static void
2376 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2377 {
2378         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2379         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2380
2381         int_map |= FM10K_MISC_VEC_ID;
2382
2383         /* Only INT 0 available, other 15 are reserved. */
2384         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2385
2386         /* Disable ITR 0 */
2387         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2388         FM10K_WRITE_FLUSH(hw);
2389 }
2390
2391 static int
2392 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2393 {
2394         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2395         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2396
2397         /* Enable ITR */
2398         if (hw->mac.type == fm10k_mac_pf)
2399                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2400                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2401         else
2402                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2403                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2404         rte_intr_ack(&pdev->intr_handle);
2405         return 0;
2406 }
2407
2408 static int
2409 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2410 {
2411         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2412         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2413
2414         /* Disable ITR */
2415         if (hw->mac.type == fm10k_mac_pf)
2416                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2417                         FM10K_ITR_MASK_SET);
2418         else
2419                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2420                         FM10K_ITR_MASK_SET);
2421         return 0;
2422 }
2423
2424 static int
2425 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2426 {
2427         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2428         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2429         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2430         uint32_t intr_vector, vec;
2431         uint16_t queue_id;
2432         int result = 0;
2433
2434         /* fm10k needs one separate interrupt for mailbox,
2435          * so only drivers which support multiple interrupt vectors
2436          * e.g. vfio-pci can work for fm10k interrupt mode
2437          */
2438         if (!rte_intr_cap_multiple(intr_handle) ||
2439                         dev->data->dev_conf.intr_conf.rxq == 0)
2440                 return result;
2441
2442         intr_vector = dev->data->nb_rx_queues;
2443
2444         /* disable interrupt first */
2445         rte_intr_disable(intr_handle);
2446         if (hw->mac.type == fm10k_mac_pf)
2447                 fm10k_dev_disable_intr_pf(dev);
2448         else
2449                 fm10k_dev_disable_intr_vf(dev);
2450
2451         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2452                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2453                 result = -EIO;
2454         }
2455
2456         if (rte_intr_dp_is_en(intr_handle) && !result) {
2457                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2458                         dev->data->nb_rx_queues * sizeof(int), 0);
2459                 if (intr_handle->intr_vec) {
2460                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2461                                         queue_id < dev->data->nb_rx_queues;
2462                                         queue_id++) {
2463                                 intr_handle->intr_vec[queue_id] = vec;
2464                                 if (vec < intr_handle->nb_efd - 1
2465                                                 + FM10K_RX_VEC_START)
2466                                         vec++;
2467                         }
2468                 } else {
2469                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2470                                 " intr_vec", dev->data->nb_rx_queues);
2471                         rte_intr_efd_disable(intr_handle);
2472                         result = -ENOMEM;
2473                 }
2474         }
2475
2476         if (hw->mac.type == fm10k_mac_pf)
2477                 fm10k_dev_enable_intr_pf(dev);
2478         else
2479                 fm10k_dev_enable_intr_vf(dev);
2480         rte_intr_enable(intr_handle);
2481         hw->mac.ops.update_int_moderator(hw);
2482         return result;
2483 }
2484
2485 static int
2486 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2487 {
2488         struct fm10k_fault fault;
2489         int err;
2490         const char *estr = "Unknown error";
2491
2492         /* Process PCA fault */
2493         if (eicr & FM10K_EICR_PCA_FAULT) {
2494                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2495                 if (err)
2496                         goto error;
2497                 switch (fault.type) {
2498                 case PCA_NO_FAULT:
2499                         estr = "PCA_NO_FAULT"; break;
2500                 case PCA_UNMAPPED_ADDR:
2501                         estr = "PCA_UNMAPPED_ADDR"; break;
2502                 case PCA_BAD_QACCESS_PF:
2503                         estr = "PCA_BAD_QACCESS_PF"; break;
2504                 case PCA_BAD_QACCESS_VF:
2505                         estr = "PCA_BAD_QACCESS_VF"; break;
2506                 case PCA_MALICIOUS_REQ:
2507                         estr = "PCA_MALICIOUS_REQ"; break;
2508                 case PCA_POISONED_TLP:
2509                         estr = "PCA_POISONED_TLP"; break;
2510                 case PCA_TLP_ABORT:
2511                         estr = "PCA_TLP_ABORT"; break;
2512                 default:
2513                         goto error;
2514                 }
2515                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2516                         estr, fault.func ? "VF" : "PF", fault.func,
2517                         fault.address, fault.specinfo);
2518         }
2519
2520         /* Process THI fault */
2521         if (eicr & FM10K_EICR_THI_FAULT) {
2522                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2523                 if (err)
2524                         goto error;
2525                 switch (fault.type) {
2526                 case THI_NO_FAULT:
2527                         estr = "THI_NO_FAULT"; break;
2528                 case THI_MAL_DIS_Q_FAULT:
2529                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2530                 default:
2531                         goto error;
2532                 }
2533                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2534                         estr, fault.func ? "VF" : "PF", fault.func,
2535                         fault.address, fault.specinfo);
2536         }
2537
2538         /* Process FUM fault */
2539         if (eicr & FM10K_EICR_FUM_FAULT) {
2540                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2541                 if (err)
2542                         goto error;
2543                 switch (fault.type) {
2544                 case FUM_NO_FAULT:
2545                         estr = "FUM_NO_FAULT"; break;
2546                 case FUM_UNMAPPED_ADDR:
2547                         estr = "FUM_UNMAPPED_ADDR"; break;
2548                 case FUM_POISONED_TLP:
2549                         estr = "FUM_POISONED_TLP"; break;
2550                 case FUM_BAD_VF_QACCESS:
2551                         estr = "FUM_BAD_VF_QACCESS"; break;
2552                 case FUM_ADD_DECODE_ERR:
2553                         estr = "FUM_ADD_DECODE_ERR"; break;
2554                 case FUM_RO_ERROR:
2555                         estr = "FUM_RO_ERROR"; break;
2556                 case FUM_QPRC_CRC_ERROR:
2557                         estr = "FUM_QPRC_CRC_ERROR"; break;
2558                 case FUM_CSR_TIMEOUT:
2559                         estr = "FUM_CSR_TIMEOUT"; break;
2560                 case FUM_INVALID_TYPE:
2561                         estr = "FUM_INVALID_TYPE"; break;
2562                 case FUM_INVALID_LENGTH:
2563                         estr = "FUM_INVALID_LENGTH"; break;
2564                 case FUM_INVALID_BE:
2565                         estr = "FUM_INVALID_BE"; break;
2566                 case FUM_INVALID_ALIGN:
2567                         estr = "FUM_INVALID_ALIGN"; break;
2568                 default:
2569                         goto error;
2570                 }
2571                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2572                         estr, fault.func ? "VF" : "PF", fault.func,
2573                         fault.address, fault.specinfo);
2574         }
2575
2576         return 0;
2577 error:
2578         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2579         return err;
2580 }
2581
2582 /**
2583  * PF interrupt handler triggered by NIC for handling specific interrupt.
2584  *
2585  * @param handle
2586  *  Pointer to interrupt handle.
2587  * @param param
2588  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2589  *
2590  * @return
2591  *  void
2592  */
2593 static void
2594 fm10k_dev_interrupt_handler_pf(void *param)
2595 {
2596         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2597         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2598         uint32_t cause, status;
2599         struct fm10k_dev_info *dev_info =
2600                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2601         int status_mbx;
2602         s32 err;
2603
2604         if (hw->mac.type != fm10k_mac_pf)
2605                 return;
2606
2607         cause = FM10K_READ_REG(hw, FM10K_EICR);
2608
2609         /* Handle PCI fault cases */
2610         if (cause & FM10K_EICR_FAULT_MASK) {
2611                 PMD_INIT_LOG(ERR, "INT: find fault!");
2612                 fm10k_dev_handle_fault(hw, cause);
2613         }
2614
2615         /* Handle switch up/down */
2616         if (cause & FM10K_EICR_SWITCHNOTREADY)
2617                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2618
2619         if (cause & FM10K_EICR_SWITCHREADY) {
2620                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2621                 if (dev_info->sm_down == 1) {
2622                         fm10k_mbx_lock(hw);
2623
2624                         /* For recreating logical ports */
2625                         status_mbx = hw->mac.ops.update_lport_state(hw,
2626                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2627                         if (status_mbx == FM10K_SUCCESS)
2628                                 PMD_INIT_LOG(INFO,
2629                                         "INT: Recreated Logical port");
2630                         else
2631                                 PMD_INIT_LOG(INFO,
2632                                         "INT: Logical ports weren't recreated");
2633
2634                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2635                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2636                         if (status_mbx != FM10K_SUCCESS)
2637                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2638
2639                         fm10k_mbx_unlock(hw);
2640
2641                         /* first clear the internal SW recording structure */
2642                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2643                                                 ETH_MQ_RX_VMDQ_FLAG))
2644                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2645                                         false);
2646
2647                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2648                                         MAIN_VSI_POOL_NUMBER);
2649
2650                         /*
2651                          * Add default mac address and vlan for the logical
2652                          * ports that have been created, leave to the
2653                          * application to fully recover Rx filtering.
2654                          */
2655                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2656                                         MAIN_VSI_POOL_NUMBER);
2657
2658                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2659                                                 ETH_MQ_RX_VMDQ_FLAG))
2660                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2661                                         true);
2662
2663                         dev_info->sm_down = 0;
2664                         _rte_eth_dev_callback_process(dev,
2665                                         RTE_ETH_EVENT_INTR_LSC,
2666                                         NULL);
2667                 }
2668         }
2669
2670         /* Handle mailbox message */
2671         fm10k_mbx_lock(hw);
2672         err = hw->mbx.ops.process(hw, &hw->mbx);
2673         fm10k_mbx_unlock(hw);
2674
2675         if (err == FM10K_ERR_RESET_REQUESTED) {
2676                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2677                 dev_info->sm_down = 1;
2678                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2679                                 NULL);
2680         }
2681
2682         /* Handle SRAM error */
2683         if (cause & FM10K_EICR_SRAMERROR) {
2684                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2685
2686                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2687                 /* Write to clear pending bits */
2688                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2689
2690                 /* Todo: print out error message after shared code  updates */
2691         }
2692
2693         /* Clear these 3 events if having any */
2694         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2695                  FM10K_EICR_SWITCHREADY;
2696         if (cause)
2697                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2698
2699         /* Re-enable interrupt from device side */
2700         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2701                                         FM10K_ITR_MASK_CLEAR);
2702         /* Re-enable interrupt from host side */
2703         rte_intr_ack(dev->intr_handle);
2704 }
2705
2706 /**
2707  * VF interrupt handler triggered by NIC for handling specific interrupt.
2708  *
2709  * @param handle
2710  *  Pointer to interrupt handle.
2711  * @param param
2712  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2713  *
2714  * @return
2715  *  void
2716  */
2717 static void
2718 fm10k_dev_interrupt_handler_vf(void *param)
2719 {
2720         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2721         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2722         struct fm10k_mbx_info *mbx = &hw->mbx;
2723         struct fm10k_dev_info *dev_info =
2724                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2725         const enum fm10k_mbx_state state = mbx->state;
2726         int status_mbx;
2727
2728         if (hw->mac.type != fm10k_mac_vf)
2729                 return;
2730
2731         /* Handle mailbox message if lock is acquired */
2732         fm10k_mbx_lock(hw);
2733         hw->mbx.ops.process(hw, &hw->mbx);
2734         fm10k_mbx_unlock(hw);
2735
2736         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2737                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2738
2739                 fm10k_mbx_lock(hw);
2740                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2741                                 MAX_LPORT_NUM, 1);
2742                 fm10k_mbx_unlock(hw);
2743
2744                 /* Setting reset flag */
2745                 dev_info->sm_down = 1;
2746                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2747                                 NULL);
2748         }
2749
2750         if (dev_info->sm_down == 1 &&
2751                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2752                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2753                 fm10k_mbx_lock(hw);
2754                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2755                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2756                 if (status_mbx != FM10K_SUCCESS)
2757                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2758                 fm10k_mbx_unlock(hw);
2759
2760                 /* first clear the internal SW recording structure */
2761                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2762                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2763                                 MAIN_VSI_POOL_NUMBER);
2764
2765                 /*
2766                  * Add default mac address and vlan for the logical ports that
2767                  * have been created, leave to the application to fully recover
2768                  * Rx filtering.
2769                  */
2770                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2771                                 MAIN_VSI_POOL_NUMBER);
2772                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2773
2774                 dev_info->sm_down = 0;
2775                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2776                                 NULL);
2777         }
2778
2779         /* Re-enable interrupt from device side */
2780         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2781                                         FM10K_ITR_MASK_CLEAR);
2782         /* Re-enable interrupt from host side */
2783         rte_intr_ack(dev->intr_handle);
2784 }
2785
2786 /* Mailbox message handler in VF */
2787 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2788         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2789         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2790         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2791         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2792 };
2793
2794 static int
2795 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2796 {
2797         int err = 0;
2798
2799         /* Initialize mailbox lock */
2800         fm10k_mbx_initlock(hw);
2801
2802         /* Replace default message handler with new ones */
2803         if (hw->mac.type == fm10k_mac_vf)
2804                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2805
2806         if (err) {
2807                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2808                                 err);
2809                 return err;
2810         }
2811         /* Connect to SM for PF device or PF for VF device */
2812         return hw->mbx.ops.connect(hw, &hw->mbx);
2813 }
2814
2815 static void
2816 fm10k_close_mbx_service(struct fm10k_hw *hw)
2817 {
2818         /* Disconnect from SM for PF device or PF for VF device */
2819         hw->mbx.ops.disconnect(hw, &hw->mbx);
2820 }
2821
2822 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2823         .dev_configure          = fm10k_dev_configure,
2824         .dev_start              = fm10k_dev_start,
2825         .dev_stop               = fm10k_dev_stop,
2826         .dev_close              = fm10k_dev_close,
2827         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2828         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2829         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2830         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2831         .stats_get              = fm10k_stats_get,
2832         .xstats_get             = fm10k_xstats_get,
2833         .xstats_get_names       = fm10k_xstats_get_names,
2834         .stats_reset            = fm10k_stats_reset,
2835         .xstats_reset           = fm10k_stats_reset,
2836         .link_update            = fm10k_link_update,
2837         .dev_infos_get          = fm10k_dev_infos_get,
2838         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2839         .vlan_filter_set        = fm10k_vlan_filter_set,
2840         .vlan_offload_set       = fm10k_vlan_offload_set,
2841         .mac_addr_add           = fm10k_macaddr_add,
2842         .mac_addr_remove        = fm10k_macaddr_remove,
2843         .rx_queue_start         = fm10k_dev_rx_queue_start,
2844         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2845         .tx_queue_start         = fm10k_dev_tx_queue_start,
2846         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2847         .rx_queue_setup         = fm10k_rx_queue_setup,
2848         .rx_queue_release       = fm10k_rx_queue_release,
2849         .tx_queue_setup         = fm10k_tx_queue_setup,
2850         .tx_queue_release       = fm10k_tx_queue_release,
2851         .rx_queue_count         = fm10k_dev_rx_queue_count,
2852         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2853         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2854         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2855         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2856         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2857         .reta_update            = fm10k_reta_update,
2858         .reta_query             = fm10k_reta_query,
2859         .rss_hash_update        = fm10k_rss_hash_update,
2860         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2861 };
2862
2863 static int ftag_check_handler(__rte_unused const char *key,
2864                 const char *value, __rte_unused void *opaque)
2865 {
2866         if (strcmp(value, "1"))
2867                 return -1;
2868
2869         return 0;
2870 }
2871
2872 static int
2873 fm10k_check_ftag(struct rte_devargs *devargs)
2874 {
2875         struct rte_kvargs *kvlist;
2876         const char *ftag_key = "enable_ftag";
2877
2878         if (devargs == NULL)
2879                 return 0;
2880
2881         kvlist = rte_kvargs_parse(devargs->args, NULL);
2882         if (kvlist == NULL)
2883                 return 0;
2884
2885         if (!rte_kvargs_count(kvlist, ftag_key)) {
2886                 rte_kvargs_free(kvlist);
2887                 return 0;
2888         }
2889         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2890         if (rte_kvargs_process(kvlist, ftag_key,
2891                                 ftag_check_handler, NULL) < 0) {
2892                 rte_kvargs_free(kvlist);
2893                 return 0;
2894         }
2895         rte_kvargs_free(kvlist);
2896
2897         return 1;
2898 }
2899
2900 static uint16_t
2901 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2902                     uint16_t nb_pkts)
2903 {
2904         uint16_t nb_tx = 0;
2905         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2906
2907         while (nb_pkts) {
2908                 uint16_t ret, num;
2909
2910                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2911                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2912                                                  num);
2913                 nb_tx += ret;
2914                 nb_pkts -= ret;
2915                 if (ret < num)
2916                         break;
2917         }
2918
2919         return nb_tx;
2920 }
2921
2922 static void __attribute__((cold))
2923 fm10k_set_tx_function(struct rte_eth_dev *dev)
2924 {
2925         struct fm10k_tx_queue *txq;
2926         int i;
2927         int use_sse = 1;
2928         uint16_t tx_ftag_en = 0;
2929
2930         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2931                 /* primary process has set the ftag flag and offloads */
2932                 txq = dev->data->tx_queues[0];
2933                 if (fm10k_tx_vec_condition_check(txq)) {
2934                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2935                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2936                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2937                 } else {
2938                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2939                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2940                         dev->tx_pkt_prepare = NULL;
2941                 }
2942                 return;
2943         }
2944
2945         if (fm10k_check_ftag(dev->device->devargs))
2946                 tx_ftag_en = 1;
2947
2948         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2949                 txq = dev->data->tx_queues[i];
2950                 txq->tx_ftag_en = tx_ftag_en;
2951                 /* Check if Vector Tx is satisfied */
2952                 if (fm10k_tx_vec_condition_check(txq))
2953                         use_sse = 0;
2954         }
2955
2956         if (use_sse) {
2957                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2958                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2959                         txq = dev->data->tx_queues[i];
2960                         fm10k_txq_vec_setup(txq);
2961                 }
2962                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2963                 dev->tx_pkt_prepare = NULL;
2964         } else {
2965                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2966                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2967                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2968         }
2969 }
2970
2971 static void __attribute__((cold))
2972 fm10k_set_rx_function(struct rte_eth_dev *dev)
2973 {
2974         struct fm10k_dev_info *dev_info =
2975                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2976         uint16_t i, rx_using_sse;
2977         uint16_t rx_ftag_en = 0;
2978
2979         if (fm10k_check_ftag(dev->device->devargs))
2980                 rx_ftag_en = 1;
2981
2982         /* In order to allow Vector Rx there are a few configuration
2983          * conditions to be met.
2984          */
2985         if (!fm10k_rx_vec_condition_check(dev) &&
2986                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2987                 if (dev->data->scattered_rx)
2988                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2989                 else
2990                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2991         } else if (dev->data->scattered_rx)
2992                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2993         else
2994                 dev->rx_pkt_burst = fm10k_recv_pkts;
2995
2996         rx_using_sse =
2997                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2998                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2999
3000         if (rx_using_sse)
3001                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3002         else
3003                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3004
3005         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3006                 return;
3007
3008         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3009                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3010
3011                 rxq->rx_using_sse = rx_using_sse;
3012                 rxq->rx_ftag_en = rx_ftag_en;
3013         }
3014 }
3015
3016 static void
3017 fm10k_params_init(struct rte_eth_dev *dev)
3018 {
3019         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3020         struct fm10k_dev_info *info =
3021                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3022
3023         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3024          * there is no way to get link status without reading BAR4.  Until this
3025          * works, assume we have maximum bandwidth.
3026          * @todo - fix bus info
3027          */
3028         hw->bus_caps.speed = fm10k_bus_speed_8000;
3029         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3030         hw->bus_caps.payload = fm10k_bus_payload_512;
3031         hw->bus.speed = fm10k_bus_speed_8000;
3032         hw->bus.width = fm10k_bus_width_pcie_x8;
3033         hw->bus.payload = fm10k_bus_payload_256;
3034
3035         info->rx_vec_allowed = true;
3036         info->sm_down = false;
3037 }
3038
3039 static int
3040 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3041 {
3042         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3043         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3044         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3045         int diag, i;
3046         struct fm10k_macvlan_filter_info *macvlan;
3047
3048         PMD_INIT_FUNC_TRACE();
3049
3050         dev->dev_ops = &fm10k_eth_dev_ops;
3051         dev->rx_pkt_burst = &fm10k_recv_pkts;
3052         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3053         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3054
3055         /*
3056          * Primary process does the whole initialization, for secondary
3057          * processes, we just select the same Rx and Tx function as primary.
3058          */
3059         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3060                 fm10k_set_rx_function(dev);
3061                 fm10k_set_tx_function(dev);
3062                 return 0;
3063         }
3064
3065         rte_eth_copy_pci_info(dev, pdev);
3066
3067         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3068         memset(macvlan, 0, sizeof(*macvlan));
3069         /* Vendor and Device ID need to be set before init of shared code */
3070         memset(hw, 0, sizeof(*hw));
3071         hw->device_id = pdev->id.device_id;
3072         hw->vendor_id = pdev->id.vendor_id;
3073         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3074         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3075         hw->revision_id = 0;
3076         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3077         if (hw->hw_addr == NULL) {
3078                 PMD_INIT_LOG(ERR, "Bad mem resource."
3079                         " Try to blacklist unused devices.");
3080                 return -EIO;
3081         }
3082
3083         /* Store fm10k_adapter pointer */
3084         hw->back = dev->data->dev_private;
3085
3086         /* Initialize the shared code */
3087         diag = fm10k_init_shared_code(hw);
3088         if (diag != FM10K_SUCCESS) {
3089                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3090                 return -EIO;
3091         }
3092
3093         /* Initialize parameters */
3094         fm10k_params_init(dev);
3095
3096         /* Initialize the hw */
3097         diag = fm10k_init_hw(hw);
3098         if (diag != FM10K_SUCCESS) {
3099                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3100                 return -EIO;
3101         }
3102
3103         /* Initialize MAC address(es) */
3104         dev->data->mac_addrs = rte_zmalloc("fm10k",
3105                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3106         if (dev->data->mac_addrs == NULL) {
3107                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3108                 return -ENOMEM;
3109         }
3110
3111         diag = fm10k_read_mac_addr(hw);
3112
3113         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3114                         &dev->data->mac_addrs[0]);
3115
3116         if (diag != FM10K_SUCCESS ||
3117                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3118
3119                 /* Generate a random addr */
3120                 rte_eth_random_addr(hw->mac.addr);
3121                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3122                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3123                 &dev->data->mac_addrs[0]);
3124         }
3125
3126         /* Reset the hw statistics */
3127         diag = fm10k_stats_reset(dev);
3128         if (diag != 0) {
3129                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3130                 return diag;
3131         }
3132
3133         /* Reset the hw */
3134         diag = fm10k_reset_hw(hw);
3135         if (diag != FM10K_SUCCESS) {
3136                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3137                 return -EIO;
3138         }
3139
3140         /* Setup mailbox service */
3141         diag = fm10k_setup_mbx_service(hw);
3142         if (diag != FM10K_SUCCESS) {
3143                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3144                 return -EIO;
3145         }
3146
3147         /*PF/VF has different interrupt handling mechanism */
3148         if (hw->mac.type == fm10k_mac_pf) {
3149                 /* register callback func to eal lib */
3150                 rte_intr_callback_register(intr_handle,
3151                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3152
3153                 /* enable MISC interrupt */
3154                 fm10k_dev_enable_intr_pf(dev);
3155         } else { /* VF */
3156                 rte_intr_callback_register(intr_handle,
3157                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3158
3159                 fm10k_dev_enable_intr_vf(dev);
3160         }
3161
3162         /* Enable intr after callback registered */
3163         rte_intr_enable(intr_handle);
3164
3165         hw->mac.ops.update_int_moderator(hw);
3166
3167         /* Make sure Switch Manager is ready before going forward. */
3168         if (hw->mac.type == fm10k_mac_pf) {
3169                 int switch_ready = 0;
3170
3171                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3172                         fm10k_mbx_lock(hw);
3173                         hw->mac.ops.get_host_state(hw, &switch_ready);
3174                         fm10k_mbx_unlock(hw);
3175                         if (switch_ready)
3176                                 break;
3177                         /* Delay some time to acquire async LPORT_MAP info. */
3178                         rte_delay_us(WAIT_SWITCH_MSG_US);
3179                 }
3180
3181                 if (switch_ready == 0) {
3182                         PMD_INIT_LOG(ERR, "switch is not ready");
3183                         return -1;
3184                 }
3185         }
3186
3187         /*
3188          * Below function will trigger operations on mailbox, acquire lock to
3189          * avoid race condition from interrupt handler. Operations on mailbox
3190          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3191          * will handle and generate an interrupt to our side. Then,  FIFO in
3192          * mailbox will be touched.
3193          */
3194         fm10k_mbx_lock(hw);
3195         /* Enable port first */
3196         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3197                                         MAX_LPORT_NUM, 1);
3198
3199         /* Set unicast mode by default. App can change to other mode in other
3200          * API func.
3201          */
3202         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3203                                         FM10K_XCAST_MODE_NONE);
3204
3205         fm10k_mbx_unlock(hw);
3206
3207         /* Make sure default VID is ready before going forward. */
3208         if (hw->mac.type == fm10k_mac_pf) {
3209                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3210                         if (hw->mac.default_vid)
3211                                 break;
3212                         /* Delay some time to acquire async port VLAN info. */
3213                         rte_delay_us(WAIT_SWITCH_MSG_US);
3214                 }
3215
3216                 if (!hw->mac.default_vid) {
3217                         PMD_INIT_LOG(ERR, "default VID is not ready");
3218                         return -1;
3219                 }
3220         }
3221
3222         /* Add default mac address */
3223         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3224                 MAIN_VSI_POOL_NUMBER);
3225
3226         return 0;
3227 }
3228
3229 static int
3230 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3231 {
3232         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3233         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3234         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3235         PMD_INIT_FUNC_TRACE();
3236
3237         /* only uninitialize in the primary process */
3238         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3239                 return 0;
3240
3241         /* safe to close dev here */
3242         fm10k_dev_close(dev);
3243
3244         dev->dev_ops = NULL;
3245         dev->rx_pkt_burst = NULL;
3246         dev->tx_pkt_burst = NULL;
3247
3248         /* disable uio/vfio intr */
3249         rte_intr_disable(intr_handle);
3250
3251         /*PF/VF has different interrupt handling mechanism */
3252         if (hw->mac.type == fm10k_mac_pf) {
3253                 /* disable interrupt */
3254                 fm10k_dev_disable_intr_pf(dev);
3255
3256                 /* unregister callback func to eal lib */
3257                 rte_intr_callback_unregister(intr_handle,
3258                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3259         } else {
3260                 /* disable interrupt */
3261                 fm10k_dev_disable_intr_vf(dev);
3262
3263                 rte_intr_callback_unregister(intr_handle,
3264                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3265         }
3266
3267         return 0;
3268 }
3269
3270 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3271         struct rte_pci_device *pci_dev)
3272 {
3273         return rte_eth_dev_pci_generic_probe(pci_dev,
3274                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3275 }
3276
3277 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3278 {
3279         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3280 }
3281
3282 /*
3283  * The set of PCI devices this driver supports. This driver will enable both PF
3284  * and SRIOV-VF devices.
3285  */
3286 static const struct rte_pci_id pci_id_fm10k_map[] = {
3287         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3288         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3289         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3290         { .vendor_id = 0, /* sentinel */ },
3291 };
3292
3293 static struct rte_pci_driver rte_pmd_fm10k = {
3294         .id_table = pci_id_fm10k_map,
3295         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3296         .probe = eth_fm10k_pci_probe,
3297         .remove = eth_fm10k_pci_remove,
3298 };
3299
3300 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3301 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3302 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3303
3304 RTE_INIT(fm10k_init_log)
3305 {
3306         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3307         if (fm10k_logtype_init >= 0)
3308                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3309         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3310         if (fm10k_logtype_driver >= 0)
3311                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3312 }