drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
  47 int fm10k_logtype_rx;
  48 #endif
  49 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
  50 int fm10k_logtype_tx;
  51 #endif
  52 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
  53 int fm10k_logtype_tx_free;
  54 #endif
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  65         const u8 *mac, bool add, uint32_t pool);
  66 static void fm10k_tx_queue_release(void *queue);
  67 static void fm10k_rx_queue_release(void *queue);
  68 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  69 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  70 static int fm10k_check_ftag(struct rte_devargs *devargs);
  71 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  72
  73 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  74                                struct rte_eth_dev_info *dev_info);
  75 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  76 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  77 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  78 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  79
  80 struct fm10k_xstats_name_off {
  81         char name[RTE_ETH_XSTATS_NAME_SIZE];
  82         unsigned offset;
  83 };
  84
  85 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  86         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  87         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  88         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  89         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  90         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  91         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  92         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  93         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  94                 nodesc_drop)},
  95 };
  96
  97 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  98                 sizeof(fm10k_hw_stats_strings[0]))
  99
 100 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
 101         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
 102         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
 103         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
 104 };
 105
 106 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
 107                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
 108
 109 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 110         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 111         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 112 };
 113
 114 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 115                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 116
 117 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 118                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 119 static int
 120 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 121
 122 static void
 123 fm10k_mbx_initlock(struct fm10k_hw *hw)
 124 {
 125         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 126 }
 127
 128 static void
 129 fm10k_mbx_lock(struct fm10k_hw *hw)
 130 {
 131         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 132                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 133 }
 134
 135 static void
 136 fm10k_mbx_unlock(struct fm10k_hw *hw)
 137 {
 138         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 139 }
 140
 141 /* Stubs needed for linkage when vPMD is disabled */
 142 __rte_weak int
 143 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 144 {
 145         return -1;
 146 }
 147
 148 __rte_weak uint16_t
 149 fm10k_recv_pkts_vec(
 150         __rte_unused void *rx_queue,
 151         __rte_unused struct rte_mbuf **rx_pkts,
 152         __rte_unused uint16_t nb_pkts)
 153 {
 154         return 0;
 155 }
 156
 157 __rte_weak uint16_t
 158 fm10k_recv_scattered_pkts_vec(
 159                 __rte_unused void *rx_queue,
 160                 __rte_unused struct rte_mbuf **rx_pkts,
 161                 __rte_unused uint16_t nb_pkts)
 162 {
 163         return 0;
 164 }
 165
 166 __rte_weak int
 167 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 168
 169 {
 170         return -1;
 171 }
 172
 173 __rte_weak void
 174 fm10k_rx_queue_release_mbufs_vec(
 175                 __rte_unused struct fm10k_rx_queue *rxq)
 176 {
 177         return;
 178 }
 179
 180 __rte_weak void
 181 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 182 {
 183         return;
 184 }
 185
 186 __rte_weak int
 187 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 188 {
 189         return -1;
 190 }
 191
 192 __rte_weak uint16_t
 193 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 194                            __rte_unused struct rte_mbuf **tx_pkts,
 195                            __rte_unused uint16_t nb_pkts)
 196 {
 197         return 0;
 198 }
 199
 200 /*
 201  * reset queue to initial state, allocate software buffers used when starting
 202  * device.
 203  * return 0 on success
 204  * return -ENOMEM if buffers cannot be allocated
 205  * return -EINVAL if buffers do not satisfy alignment condition
 206  */
 207 static inline int
 208 rx_queue_reset(struct fm10k_rx_queue *q)
 209 {
 210         static const union fm10k_rx_desc zero = {{0} };
 211         uint64_t dma_addr;
 212         int i, diag;
 213         PMD_INIT_FUNC_TRACE();
 214
 215         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 216         if (diag != 0)
 217                 return -ENOMEM;
 218
 219         for (i = 0; i < q->nb_desc; ++i) {
 220                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 221                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 222                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 223                                                 q->nb_desc);
 224                         return -EINVAL;
 225                 }
 226                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 227                 q->hw_ring[i].q.pkt_addr = dma_addr;
 228                 q->hw_ring[i].q.hdr_addr = dma_addr;
 229         }
 230
 231         /* initialize extra software ring entries. Space for these extra
 232          * entries is always allocated.
 233          */
 234         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 235         for (i = 0; i < q->nb_fake_desc; ++i) {
 236                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 237                 q->hw_ring[q->nb_desc + i] = zero;
 238         }
 239
 240         q->next_dd = 0;
 241         q->next_alloc = 0;
 242         q->next_trigger = q->alloc_thresh - 1;
 243         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 244         q->rxrearm_start = 0;
 245         q->rxrearm_nb = 0;
 246
 247         return 0;
 248 }
 249
 250 /*
 251  * clean queue, descriptor rings, free software buffers used when stopping
 252  * device.
 253  */
 254 static inline void
 255 rx_queue_clean(struct fm10k_rx_queue *q)
 256 {
 257         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 258         uint32_t i;
 259         PMD_INIT_FUNC_TRACE();
 260
 261         /* zero descriptor rings */
 262         for (i = 0; i < q->nb_desc; ++i)
 263                 q->hw_ring[i] = zero;
 264
 265         /* zero faked descriptors */
 266         for (i = 0; i < q->nb_fake_desc; ++i)
 267                 q->hw_ring[q->nb_desc + i] = zero;
 268
 269         /* vPMD driver has a different way of releasing mbufs. */
 270         if (q->rx_using_sse) {
 271                 fm10k_rx_queue_release_mbufs_vec(q);
 272                 return;
 273         }
 274
 275         /* free software buffers */
 276         for (i = 0; i < q->nb_desc; ++i) {
 277                 if (q->sw_ring[i]) {
 278                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 279                         q->sw_ring[i] = NULL;
 280                 }
 281         }
 282 }
 283
 284 /*
 285  * free all queue memory used when releasing the queue (i.e. configure)
 286  */
 287 static inline void
 288 rx_queue_free(struct fm10k_rx_queue *q)
 289 {
 290         PMD_INIT_FUNC_TRACE();
 291         if (q) {
 292                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 293                 rx_queue_clean(q);
 294                 if (q->sw_ring) {
 295                         rte_free(q->sw_ring);
 296                         q->sw_ring = NULL;
 297                 }
 298                 rte_free(q);
 299                 q = NULL;
 300         }
 301 }
 302
 303 /*
 304  * disable RX queue, wait unitl HW finished necessary flush operation
 305  */
 306 static inline int
 307 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 308 {
 309         uint32_t reg, i;
 310
 311         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 312         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 313                         reg & ~FM10K_RXQCTL_ENABLE);
 314
 315         /* Wait 100us at most */
 316         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 317                 rte_delay_us(1);
 318                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 319                 if (!(reg & FM10K_RXQCTL_ENABLE))
 320                         break;
 321         }
 322
 323         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 324                 return -1;
 325
 326         return 0;
 327 }
 328
 329 /*
 330  * reset queue to initial state, allocate software buffers used when starting
 331  * device
 332  */
 333 static inline void
 334 tx_queue_reset(struct fm10k_tx_queue *q)
 335 {
 336         PMD_INIT_FUNC_TRACE();
 337         q->last_free = 0;
 338         q->next_free = 0;
 339         q->nb_used = 0;
 340         q->nb_free = q->nb_desc - 1;
 341         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 342         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 343 }
 344
 345 /*
 346  * clean queue, descriptor rings, free software buffers used when stopping
 347  * device
 348  */
 349 static inline void
 350 tx_queue_clean(struct fm10k_tx_queue *q)
 351 {
 352         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 353         uint32_t i;
 354         PMD_INIT_FUNC_TRACE();
 355
 356         /* zero descriptor rings */
 357         for (i = 0; i < q->nb_desc; ++i)
 358                 q->hw_ring[i] = zero;
 359
 360         /* free software buffers */
 361         for (i = 0; i < q->nb_desc; ++i) {
 362                 if (q->sw_ring[i]) {
 363                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 364                         q->sw_ring[i] = NULL;
 365                 }
 366         }
 367 }
 368
 369 /*
 370  * free all queue memory used when releasing the queue (i.e. configure)
 371  */
 372 static inline void
 373 tx_queue_free(struct fm10k_tx_queue *q)
 374 {
 375         PMD_INIT_FUNC_TRACE();
 376         if (q) {
 377                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 378                 tx_queue_clean(q);
 379                 if (q->rs_tracker.list) {
 380                         rte_free(q->rs_tracker.list);
 381                         q->rs_tracker.list = NULL;
 382                 }
 383                 if (q->sw_ring) {
 384                         rte_free(q->sw_ring);
 385                         q->sw_ring = NULL;
 386                 }
 387                 rte_free(q);
 388                 q = NULL;
 389         }
 390 }
 391
 392 /*
 393  * disable TX queue, wait unitl HW finished necessary flush operation
 394  */
 395 static inline int
 396 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 397 {
 398         uint32_t reg, i;
 399
 400         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 401         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 402                         reg & ~FM10K_TXDCTL_ENABLE);
 403
 404         /* Wait 100us at most */
 405         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 406                 rte_delay_us(1);
 407                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 408                 if (!(reg & FM10K_TXDCTL_ENABLE))
 409                         break;
 410         }
 411
 412         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 413                 return -1;
 414
 415         return 0;
 416 }
 417
 418 static int
 419 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 420 {
 421         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 422         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 423         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 424         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 425
 426         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 427
 428         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 429                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 430                 return -EINVAL;
 431         }
 432
 433         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 434                 return 0;
 435
 436         if (hw->mac.type == fm10k_mac_vf) {
 437                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 438                 return -EINVAL;
 439         }
 440
 441         /* Check VMDQ queue pool number */
 442         if (vmdq_conf->nb_queue_pools >
 443                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 444                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 445                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 446                         vmdq_conf->nb_queue_pools);
 447                 return -EINVAL;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static const struct fm10k_txq_ops def_txq_ops = {
 454         .reset = tx_queue_reset,
 455 };
 456
 457 static int
 458 fm10k_dev_configure(struct rte_eth_dev *dev)
 459 {
 460         int ret;
 461
 462         PMD_INIT_FUNC_TRACE();
 463
 464         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 465                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 466
 467         /* multipe queue mode checking */
 468         ret  = fm10k_check_mq_mode(dev);
 469         if (ret != 0) {
 470                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 471                             ret);
 472                 return ret;
 473         }
 474
 475         dev->data->scattered_rx = 0;
 476
 477         return 0;
 478 }
 479
 480 static void
 481 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 482 {
 483         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 484         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 485         uint32_t i;
 486
 487         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 488
 489         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 490                 if (!vmdq_conf->pool_map[i].pools)
 491                         continue;
 492                 fm10k_mbx_lock(hw);
 493                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 494                 fm10k_mbx_unlock(hw);
 495         }
 496 }
 497
 498 static void
 499 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 500 {
 501         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 502
 503         /* Add default mac address */
 504         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 505                 MAIN_VSI_POOL_NUMBER);
 506 }
 507
 508 static void
 509 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 510 {
 511         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 512         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 513         uint32_t mrqc, *key, i, reta, j;
 514         uint64_t hf;
 515
 516 #define RSS_KEY_SIZE 40
 517         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 518                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 519                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 520                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 521                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 522                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 523         };
 524
 525         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 526                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 527                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 528                 return;
 529         }
 530
 531         /* random key is rss_intel_key (default) or user provided (rss_key) */
 532         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 533                 key = (uint32_t *)rss_intel_key;
 534         else
 535                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 536
 537         /* Now fill our hash function seeds, 4 bytes at a time */
 538         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 539                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 540
 541         /*
 542          * Fill in redirection table
 543          * The byte-swap is needed because NIC registers are in
 544          * little-endian order.
 545          */
 546         reta = 0;
 547         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 548                 if (j == dev->data->nb_rx_queues)
 549                         j = 0;
 550                 reta = (reta << CHAR_BIT) | j;
 551                 if ((i & 3) == 3)
 552                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 553                                         rte_bswap32(reta));
 554         }
 555
 556         /*
 557          * Generate RSS hash based on packet types, TCP/UDP
 558          * port numbers and/or IPv4/v6 src and dst addresses
 559          */
 560         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 561         mrqc = 0;
 562         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 563         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 564         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 565         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 566         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 567         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 568         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 569         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 570         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 571
 572         if (mrqc == 0) {
 573                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 574                         "supported", hf);
 575                 return;
 576         }
 577
 578         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 579 }
 580
 581 static void
 582 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 583 {
 584         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 585         uint32_t i;
 586
 587         for (i = 0; i < nb_lport_new; i++) {
 588                 /* Set unicast mode by default. App can change
 589                  * to other mode in other API func.
 590                  */
 591                 fm10k_mbx_lock(hw);
 592                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 593                         FM10K_XCAST_MODE_NONE);
 594                 fm10k_mbx_unlock(hw);
 595         }
 596 }
 597
 598 static void
 599 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 600 {
 601         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 602         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 603         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 604         struct fm10k_macvlan_filter_info *macvlan;
 605         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 606         uint16_t nb_lport_new;
 607
 608         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 609         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 610
 611         fm10k_dev_rss_configure(dev);
 612
 613         /* only PF supports VMDQ */
 614         if (hw->mac.type != fm10k_mac_pf)
 615                 return;
 616
 617         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 618                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 619
 620         /* no pool number change, no need to update logic port and VLAN/MAC */
 621         if (macvlan->nb_queue_pools == nb_queue_pools)
 622                 return;
 623
 624         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 625         fm10k_dev_logic_port_update(dev, nb_lport_new);
 626
 627         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 628         memset(dev->data->mac_addrs, 0,
 629                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 630         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 631                 &dev->data->mac_addrs[0]);
 632         memset(macvlan, 0, sizeof(*macvlan));
 633         macvlan->nb_queue_pools = nb_queue_pools;
 634
 635         if (nb_queue_pools)
 636                 fm10k_dev_vmdq_rx_configure(dev);
 637         else
 638                 fm10k_dev_pf_main_vsi_reset(dev);
 639 }
 640
 641 static int
 642 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 643 {
 644         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 645         int i, ret;
 646         struct fm10k_tx_queue *txq;
 647         uint64_t base_addr;
 648         uint32_t size;
 649
 650         /* Disable TXINT to avoid possible interrupt */
 651         for (i = 0; i < hw->mac.max_queues; i++)
 652                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 653                                 3 << FM10K_TXINT_TIMER_SHIFT);
 654
 655         /* Setup TX queue */
 656         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 657                 txq = dev->data->tx_queues[i];
 658                 base_addr = txq->hw_ring_phys_addr;
 659                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 660
 661                 /* disable queue to avoid issues while updating state */
 662                 ret = tx_queue_disable(hw, i);
 663                 if (ret) {
 664                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 665                         return -1;
 666                 }
 667                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 668                  * register is read-only for VF.
 669                  */
 670                 if (fm10k_check_ftag(dev->device->devargs)) {
 671                         if (hw->mac.type == fm10k_mac_pf) {
 672                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 673                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 674                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 675                         } else {
 676                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 677                                 return -ENOTSUP;
 678                         }
 679                 }
 680
 681                 /* set location and size for descriptor ring */
 682                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 683                                 base_addr & UINT64_LOWER_32BITS_MASK);
 684                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 685                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 686                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 687
 688                 /* assign default SGLORT for each TX queue by PF */
 689                 if (hw->mac.type == fm10k_mac_pf)
 690                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 691         }
 692
 693         /* set up vector or scalar TX function as appropriate */
 694         fm10k_set_tx_function(dev);
 695
 696         return 0;
 697 }
 698
 699 static int
 700 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 701 {
 702         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 703         struct fm10k_macvlan_filter_info *macvlan;
 704         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 705         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 706         int i, ret;
 707         struct fm10k_rx_queue *rxq;
 708         uint64_t base_addr;
 709         uint32_t size;
 710         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 711         uint32_t logic_port = hw->mac.dglort_map;
 712         uint16_t buf_size;
 713         uint16_t queue_stride = 0;
 714
 715         /* enable RXINT for interrupt mode */
 716         i = 0;
 717         if (rte_intr_dp_is_en(intr_handle)) {
 718                 for (; i < dev->data->nb_rx_queues; i++) {
 719                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 720                         if (hw->mac.type == fm10k_mac_pf)
 721                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 722                                         FM10K_ITR_AUTOMASK |
 723                                         FM10K_ITR_MASK_CLEAR);
 724                         else
 725                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 726                                         FM10K_ITR_AUTOMASK |
 727                                         FM10K_ITR_MASK_CLEAR);
 728                 }
 729         }
 730         /* Disable other RXINT to avoid possible interrupt */
 731         for (; i < hw->mac.max_queues; i++)
 732                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 733                         3 << FM10K_RXINT_TIMER_SHIFT);
 734
 735         /* Setup RX queues */
 736         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 737                 rxq = dev->data->rx_queues[i];
 738                 base_addr = rxq->hw_ring_phys_addr;
 739                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 740
 741                 /* disable queue to avoid issues while updating state */
 742                 ret = rx_queue_disable(hw, i);
 743                 if (ret) {
 744                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 745                         return -1;
 746                 }
 747
 748                 /* Setup the Base and Length of the Rx Descriptor Ring */
 749                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 750                                 base_addr & UINT64_LOWER_32BITS_MASK);
 751                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 752                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 753                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 754
 755                 /* Configure the Rx buffer size for one buff without split */
 756                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 757                         RTE_PKTMBUF_HEADROOM);
 758                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 759                  * reserved for this purpose, and the worst case could be 511B.
 760                  * But SRR reg assumes all buffers have the same size. In order
 761                  * to fill the gap, we'll have to consider the worst case and
 762                  * assume 512B is reserved. If we don't do so, it's possible
 763                  * for HW to overwrite data to next mbuf.
 764                  */
 765                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 766
 767                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 768                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 769                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 770
 771                 /* It adds dual VLAN length for supporting dual VLAN */
 772                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 773                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 774                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 775                         uint32_t reg;
 776                         dev->data->scattered_rx = 1;
 777                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 778                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 779                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 780                 }
 781
 782                 /* Enable drop on empty, it's RO for VF */
 783                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 784                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 785
 786                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 787                 FM10K_WRITE_FLUSH(hw);
 788         }
 789
 790         /* Configure VMDQ/RSS if applicable */
 791         fm10k_dev_mq_rx_configure(dev);
 792
 793         /* Decide the best RX function */
 794         fm10k_set_rx_function(dev);
 795
 796         /* update RX_SGLORT for loopback suppress*/
 797         if (hw->mac.type != fm10k_mac_pf)
 798                 return 0;
 799         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 800         if (macvlan->nb_queue_pools)
 801                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 802         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 803                 if (i && queue_stride && !(i % queue_stride))
 804                         logic_port++;
 805                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 806         }
 807
 808         return 0;
 809 }
 810
 811 static int
 812 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 813 {
 814         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 815         int err;
 816         uint32_t reg;
 817         struct fm10k_rx_queue *rxq;
 818
 819         PMD_INIT_FUNC_TRACE();
 820
 821         rxq = dev->data->rx_queues[rx_queue_id];
 822         err = rx_queue_reset(rxq);
 823         if (err == -ENOMEM) {
 824                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 825                 return err;
 826         } else if (err == -EINVAL) {
 827                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 828                         " %d", err);
 829                 return err;
 830         }
 831
 832         /* Setup the HW Rx Head and Tail Descriptor Pointers
 833          * Note: this must be done AFTER the queue is enabled on real
 834          * hardware, but BEFORE the queue is enabled when using the
 835          * emulation platform. Do it in both places for now and remove
 836          * this comment and the following two register writes when the
 837          * emulation platform is no longer being used.
 838          */
 839         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 840         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 841
 842         /* Set PF ownership flag for PF devices */
 843         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 844         if (hw->mac.type == fm10k_mac_pf)
 845                 reg |= FM10K_RXQCTL_PF;
 846         reg |= FM10K_RXQCTL_ENABLE;
 847         /* enable RX queue */
 848         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 849         FM10K_WRITE_FLUSH(hw);
 850
 851         /* Setup the HW Rx Head and Tail Descriptor Pointers
 852          * Note: this must be done AFTER the queue is enabled
 853          */
 854         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 855         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 856         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 857
 858         return 0;
 859 }
 860
 861 static int
 862 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 863 {
 864         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 865
 866         PMD_INIT_FUNC_TRACE();
 867
 868         /* Disable RX queue */
 869         rx_queue_disable(hw, rx_queue_id);
 870
 871         /* Free mbuf and clean HW ring */
 872         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 873         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 874
 875         return 0;
 876 }
 877
 878 static int
 879 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 880 {
 881         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 882         /** @todo - this should be defined in the shared code */
 883 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 884         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 885         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 886
 887         PMD_INIT_FUNC_TRACE();
 888
 889         q->ops->reset(q);
 890
 891         /* reset head and tail pointers */
 892         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 893         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 894
 895         /* enable TX queue */
 896         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 897                                 FM10K_TXDCTL_ENABLE | txdctl);
 898         FM10K_WRITE_FLUSH(hw);
 899         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 900
 901         return 0;
 902 }
 903
 904 static int
 905 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 906 {
 907         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 908
 909         PMD_INIT_FUNC_TRACE();
 910
 911         tx_queue_disable(hw, tx_queue_id);
 912         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 913         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 914
 915         return 0;
 916 }
 917
 918 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 919 {
 920         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 921                 != FM10K_DGLORTMAP_NONE);
 922 }
 923
 924 static int
 925 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 926 {
 927         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 928         int status;
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         /* Return if it didn't acquire valid glort range */
 933         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 934                 return 0;
 935
 936         fm10k_mbx_lock(hw);
 937         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 938                                 FM10K_XCAST_MODE_PROMISC);
 939         fm10k_mbx_unlock(hw);
 940
 941         if (status != FM10K_SUCCESS) {
 942                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 943                 return -EAGAIN;
 944         }
 945
 946         return 0;
 947 }
 948
 949 static int
 950 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 951 {
 952         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 953         uint8_t mode;
 954         int status;
 955
 956         PMD_INIT_FUNC_TRACE();
 957
 958         /* Return if it didn't acquire valid glort range */
 959         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 960                 return 0;
 961
 962         if (dev->data->all_multicast == 1)
 963                 mode = FM10K_XCAST_MODE_ALLMULTI;
 964         else
 965                 mode = FM10K_XCAST_MODE_NONE;
 966
 967         fm10k_mbx_lock(hw);
 968         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 969                                 mode);
 970         fm10k_mbx_unlock(hw);
 971
 972         if (status != FM10K_SUCCESS) {
 973                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 974                 return -EAGAIN;
 975         }
 976
 977         return 0;
 978 }
 979
 980 static int
 981 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 982 {
 983         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 984         int status;
 985
 986         PMD_INIT_FUNC_TRACE();
 987
 988         /* Return if it didn't acquire valid glort range */
 989         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 990                 return 0;
 991
 992         /* If promiscuous mode is enabled, it doesn't make sense to enable
 993          * allmulticast and disable promiscuous since fm10k only can select
 994          * one of the modes.
 995          */
 996         if (dev->data->promiscuous) {
 997                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 998                         "needn't enable allmulticast");
 999                 return 0;
1000         }
1001
1002         fm10k_mbx_lock(hw);
1003         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1004                                 FM10K_XCAST_MODE_ALLMULTI);
1005         fm10k_mbx_unlock(hw);
1006
1007         if (status != FM10K_SUCCESS) {
1008                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1009                 return -EAGAIN;
1010         }
1011
1012         return 0;
1013 }
1014
1015 static int
1016 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1017 {
1018         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1019         int status;
1020
1021         PMD_INIT_FUNC_TRACE();
1022
1023         /* Return if it didn't acquire valid glort range */
1024         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1025                 return 0;
1026
1027         if (dev->data->promiscuous) {
1028                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1029                         "since promisc mode is enabled");
1030                 return -EINVAL;
1031         }
1032
1033         fm10k_mbx_lock(hw);
1034         /* Change mode to unicast mode */
1035         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1036                                 FM10K_XCAST_MODE_NONE);
1037         fm10k_mbx_unlock(hw);
1038
1039         if (status != FM10K_SUCCESS) {
1040                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1041                 return -EAGAIN;
1042         }
1043
1044         return 0;
1045 }
1046
1047 static void
1048 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1049 {
1050         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1051         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1052         uint16_t nb_queue_pools;
1053         struct fm10k_macvlan_filter_info *macvlan;
1054
1055         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1056         nb_queue_pools = macvlan->nb_queue_pools;
1057         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1058         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1059
1060         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1061         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1062         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1063                         hw->mac.dglort_map;
1064         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1065         /* Configure VMDQ/RSS DGlort Decoder */
1066         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1067
1068         /* Flow Director configurations, only queue number is valid. */
1069         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1070         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1071                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1072         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1074
1075         /* Invalidate all other GLORT entries */
1076         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1077                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1078                                 FM10K_DGLORTMAP_NONE);
1079 }
1080
1081 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1082 static int
1083 fm10k_dev_start(struct rte_eth_dev *dev)
1084 {
1085         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1086         int i, diag;
1087
1088         PMD_INIT_FUNC_TRACE();
1089
1090         /* stop, init, then start the hw */
1091         diag = fm10k_stop_hw(hw);
1092         if (diag != FM10K_SUCCESS) {
1093                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1094                 return -EIO;
1095         }
1096
1097         diag = fm10k_init_hw(hw);
1098         if (diag != FM10K_SUCCESS) {
1099                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1100                 return -EIO;
1101         }
1102
1103         diag = fm10k_start_hw(hw);
1104         if (diag != FM10K_SUCCESS) {
1105                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1106                 return -EIO;
1107         }
1108
1109         diag = fm10k_dev_tx_init(dev);
1110         if (diag) {
1111                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1112                 return diag;
1113         }
1114
1115         if (fm10k_dev_rxq_interrupt_setup(dev))
1116                 return -EIO;
1117
1118         diag = fm10k_dev_rx_init(dev);
1119         if (diag) {
1120                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1121                 return diag;
1122         }
1123
1124         if (hw->mac.type == fm10k_mac_pf)
1125                 fm10k_dev_dglort_map_configure(dev);
1126
1127         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1128                 struct fm10k_rx_queue *rxq;
1129                 rxq = dev->data->rx_queues[i];
1130
1131                 if (rxq->rx_deferred_start)
1132                         continue;
1133                 diag = fm10k_dev_rx_queue_start(dev, i);
1134                 if (diag != 0) {
1135                         int j;
1136                         for (j = 0; j < i; ++j)
1137                                 rx_queue_clean(dev->data->rx_queues[j]);
1138                         return diag;
1139                 }
1140         }
1141
1142         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1143                 struct fm10k_tx_queue *txq;
1144                 txq = dev->data->tx_queues[i];
1145
1146                 if (txq->tx_deferred_start)
1147                         continue;
1148                 diag = fm10k_dev_tx_queue_start(dev, i);
1149                 if (diag != 0) {
1150                         int j;
1151                         for (j = 0; j < i; ++j)
1152                                 tx_queue_clean(dev->data->tx_queues[j]);
1153                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1154                                 rx_queue_clean(dev->data->rx_queues[j]);
1155                         return diag;
1156                 }
1157         }
1158
1159         /* Update default vlan when not in VMDQ mode */
1160         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1161                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1162
1163         fm10k_link_update(dev, 0);
1164
1165         return 0;
1166 }
1167
1168 static void
1169 fm10k_dev_stop(struct rte_eth_dev *dev)
1170 {
1171         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1172         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1173         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1174         int i;
1175
1176         PMD_INIT_FUNC_TRACE();
1177
1178         if (dev->data->tx_queues)
1179                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1180                         fm10k_dev_tx_queue_stop(dev, i);
1181
1182         if (dev->data->rx_queues)
1183                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1184                         fm10k_dev_rx_queue_stop(dev, i);
1185
1186         /* Disable datapath event */
1187         if (rte_intr_dp_is_en(intr_handle)) {
1188                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1189                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1190                                 3 << FM10K_RXINT_TIMER_SHIFT);
1191                         if (hw->mac.type == fm10k_mac_pf)
1192                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1193                                         FM10K_ITR_MASK_SET);
1194                         else
1195                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1196                                         FM10K_ITR_MASK_SET);
1197                 }
1198         }
1199         /* Clean datapath event and queue/vec mapping */
1200         rte_intr_efd_disable(intr_handle);
1201         rte_free(intr_handle->intr_vec);
1202         intr_handle->intr_vec = NULL;
1203 }
1204
1205 static void
1206 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1207 {
1208         int i;
1209
1210         PMD_INIT_FUNC_TRACE();
1211
1212         if (dev->data->tx_queues) {
1213                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1214                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1215
1216                         tx_queue_free(txq);
1217                 }
1218         }
1219
1220         if (dev->data->rx_queues) {
1221                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1222                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1223         }
1224 }
1225
1226 static int
1227 fm10k_link_update(struct rte_eth_dev *dev,
1228         __rte_unused int wait_to_complete)
1229 {
1230         struct fm10k_dev_info *dev_info =
1231                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1232         PMD_INIT_FUNC_TRACE();
1233
1234         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1235         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1236         dev->data->dev_link.link_status =
1237                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1238         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1239
1240         return 0;
1241 }
1242
1243 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1244         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1245 {
1246         unsigned i, q;
1247         unsigned count = 0;
1248
1249         if (xstats_names != NULL) {
1250                 /* Note: limit checked in rte_eth_xstats_names() */
1251
1252                 /* Global stats */
1253                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1254                         snprintf(xstats_names[count].name,
1255                                 sizeof(xstats_names[count].name),
1256                                 "%s", fm10k_hw_stats_strings[count].name);
1257                         count++;
1258                 }
1259
1260                 /* PF queue stats */
1261                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1262                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1263                                 snprintf(xstats_names[count].name,
1264                                         sizeof(xstats_names[count].name),
1265                                         "rx_q%u_%s", q,
1266                                         fm10k_hw_stats_rx_q_strings[i].name);
1267                                 count++;
1268                         }
1269                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1270                                 snprintf(xstats_names[count].name,
1271                                         sizeof(xstats_names[count].name),
1272                                         "tx_q%u_%s", q,
1273                                         fm10k_hw_stats_tx_q_strings[i].name);
1274                                 count++;
1275                         }
1276                 }
1277         }
1278         return FM10K_NB_XSTATS;
1279 }
1280
1281 static int
1282 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1283                  unsigned n)
1284 {
1285         struct fm10k_hw_stats *hw_stats =
1286                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1287         unsigned i, q, count = 0;
1288
1289         if (n < FM10K_NB_XSTATS)
1290                 return FM10K_NB_XSTATS;
1291
1292         /* Global stats */
1293         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1294                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1295                         fm10k_hw_stats_strings[count].offset);
1296                 xstats[count].id = count;
1297                 count++;
1298         }
1299
1300         /* PF queue stats */
1301         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1302                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1303                         xstats[count].value =
1304                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1305                                 fm10k_hw_stats_rx_q_strings[i].offset);
1306                         xstats[count].id = count;
1307                         count++;
1308                 }
1309                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1310                         xstats[count].value =
1311                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1312                                 fm10k_hw_stats_tx_q_strings[i].offset);
1313                         xstats[count].id = count;
1314                         count++;
1315                 }
1316         }
1317
1318         return FM10K_NB_XSTATS;
1319 }
1320
1321 static int
1322 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1323 {
1324         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1325         struct fm10k_hw *hw =
1326                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1327         struct fm10k_hw_stats *hw_stats =
1328                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1329         int i;
1330
1331         PMD_INIT_FUNC_TRACE();
1332
1333         fm10k_update_hw_stats(hw, hw_stats);
1334
1335         ipackets = opackets = ibytes = obytes = imissed = 0;
1336         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1337                 (i < hw->mac.max_queues); ++i) {
1338                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1339                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1340                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1341                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1342                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1343                 ipackets += stats->q_ipackets[i];
1344                 opackets += stats->q_opackets[i];
1345                 ibytes   += stats->q_ibytes[i];
1346                 obytes   += stats->q_obytes[i];
1347                 imissed  += stats->q_errors[i];
1348         }
1349         stats->ipackets = ipackets;
1350         stats->opackets = opackets;
1351         stats->ibytes = ibytes;
1352         stats->obytes = obytes;
1353         stats->imissed = imissed;
1354         return 0;
1355 }
1356
1357 static int
1358 fm10k_stats_reset(struct rte_eth_dev *dev)
1359 {
1360         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1361         struct fm10k_hw_stats *hw_stats =
1362                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1363
1364         PMD_INIT_FUNC_TRACE();
1365
1366         memset(hw_stats, 0, sizeof(*hw_stats));
1367         fm10k_rebind_hw_stats(hw, hw_stats);
1368
1369         return 0;
1370 }
1371
1372 static int
1373 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1374         struct rte_eth_dev_info *dev_info)
1375 {
1376         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1377         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1378
1379         PMD_INIT_FUNC_TRACE();
1380
1381         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1382         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1383         dev_info->max_rx_queues      = hw->mac.max_queues;
1384         dev_info->max_tx_queues      = hw->mac.max_queues;
1385         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1386         dev_info->max_hash_mac_addrs = 0;
1387         dev_info->max_vfs            = pdev->max_vfs;
1388         dev_info->vmdq_pool_base     = 0;
1389         dev_info->vmdq_queue_base    = 0;
1390         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1391         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1392         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1393         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1394                                     dev_info->rx_queue_offload_capa;
1395         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1396         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1397                                     dev_info->tx_queue_offload_capa;
1398
1399         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1400         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1401         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1402                                         ETH_RSS_IPV6 |
1403                                         ETH_RSS_IPV6_EX |
1404                                         ETH_RSS_NONFRAG_IPV4_TCP |
1405                                         ETH_RSS_NONFRAG_IPV6_TCP |
1406                                         ETH_RSS_IPV6_TCP_EX |
1407                                         ETH_RSS_NONFRAG_IPV4_UDP |
1408                                         ETH_RSS_NONFRAG_IPV6_UDP |
1409                                         ETH_RSS_IPV6_UDP_EX;
1410
1411         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1412                 .rx_thresh = {
1413                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1414                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1415                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1416                 },
1417                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1418                 .rx_drop_en = 0,
1419                 .offloads = 0,
1420         };
1421
1422         dev_info->default_txconf = (struct rte_eth_txconf) {
1423                 .tx_thresh = {
1424                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1425                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1426                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1427                 },
1428                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1429                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1430                 .offloads = 0,
1431         };
1432
1433         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1434                 .nb_max = FM10K_MAX_RX_DESC,
1435                 .nb_min = FM10K_MIN_RX_DESC,
1436                 .nb_align = FM10K_MULT_RX_DESC,
1437         };
1438
1439         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1440                 .nb_max = FM10K_MAX_TX_DESC,
1441                 .nb_min = FM10K_MIN_TX_DESC,
1442                 .nb_align = FM10K_MULT_TX_DESC,
1443                 .nb_seg_max = FM10K_TX_MAX_SEG,
1444                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1445         };
1446
1447         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1448                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1449                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1450
1451         return 0;
1452 }
1453
1454 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1455 static const uint32_t *
1456 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1457 {
1458         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1459             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1460                 static uint32_t ptypes[] = {
1461                         /* refers to rx_desc_to_ol_flags() */
1462                         RTE_PTYPE_L2_ETHER,
1463                         RTE_PTYPE_L3_IPV4,
1464                         RTE_PTYPE_L3_IPV4_EXT,
1465                         RTE_PTYPE_L3_IPV6,
1466                         RTE_PTYPE_L3_IPV6_EXT,
1467                         RTE_PTYPE_L4_TCP,
1468                         RTE_PTYPE_L4_UDP,
1469                         RTE_PTYPE_UNKNOWN
1470                 };
1471
1472                 return ptypes;
1473         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1474                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1475                 static uint32_t ptypes_vec[] = {
1476                         /* refers to fm10k_desc_to_pktype_v() */
1477                         RTE_PTYPE_L3_IPV4,
1478                         RTE_PTYPE_L3_IPV4_EXT,
1479                         RTE_PTYPE_L3_IPV6,
1480                         RTE_PTYPE_L3_IPV6_EXT,
1481                         RTE_PTYPE_L4_TCP,
1482                         RTE_PTYPE_L4_UDP,
1483                         RTE_PTYPE_TUNNEL_GENEVE,
1484                         RTE_PTYPE_TUNNEL_NVGRE,
1485                         RTE_PTYPE_TUNNEL_VXLAN,
1486                         RTE_PTYPE_TUNNEL_GRE,
1487                         RTE_PTYPE_UNKNOWN
1488                 };
1489
1490                 return ptypes_vec;
1491         }
1492
1493         return NULL;
1494 }
1495 #else
1496 static const uint32_t *
1497 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1498 {
1499         return NULL;
1500 }
1501 #endif
1502
1503 static int
1504 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1505 {
1506         s32 result;
1507         uint16_t mac_num = 0;
1508         uint32_t vid_idx, vid_bit, mac_index;
1509         struct fm10k_hw *hw;
1510         struct fm10k_macvlan_filter_info *macvlan;
1511         struct rte_eth_dev_data *data = dev->data;
1512
1513         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1514         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1515
1516         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1517                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1518                 return -EINVAL;
1519         }
1520
1521         if (vlan_id > ETH_VLAN_ID_MAX) {
1522                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1523                 return -EINVAL;
1524         }
1525
1526         vid_idx = FM10K_VFTA_IDX(vlan_id);
1527         vid_bit = FM10K_VFTA_BIT(vlan_id);
1528         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1529         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1530                 return 0;
1531         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1532         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1533                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1534                         "in the VLAN filter table");
1535                 return -EINVAL;
1536         }
1537
1538         fm10k_mbx_lock(hw);
1539         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1540         fm10k_mbx_unlock(hw);
1541         if (result != FM10K_SUCCESS) {
1542                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1543                 return -EIO;
1544         }
1545
1546         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1547                         (result == FM10K_SUCCESS); mac_index++) {
1548                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1549                         continue;
1550                 if (mac_num > macvlan->mac_num - 1) {
1551                         PMD_INIT_LOG(ERR, "MAC address number "
1552                                         "not match");
1553                         break;
1554                 }
1555                 fm10k_mbx_lock(hw);
1556                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1557                         data->mac_addrs[mac_index].addr_bytes,
1558                         vlan_id, on, 0);
1559                 fm10k_mbx_unlock(hw);
1560                 mac_num++;
1561         }
1562         if (result != FM10K_SUCCESS) {
1563                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1564                 return -EIO;
1565         }
1566
1567         if (on) {
1568                 macvlan->vlan_num++;
1569                 macvlan->vfta[vid_idx] |= vid_bit;
1570         } else {
1571                 macvlan->vlan_num--;
1572                 macvlan->vfta[vid_idx] &= ~vid_bit;
1573         }
1574         return 0;
1575 }
1576
1577 static int
1578 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1579 {
1580         if (mask & ETH_VLAN_STRIP_MASK) {
1581                 if (!(dev->data->dev_conf.rxmode.offloads &
1582                         DEV_RX_OFFLOAD_VLAN_STRIP))
1583                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1584                                         "always on in fm10k");
1585         }
1586
1587         if (mask & ETH_VLAN_EXTEND_MASK) {
1588                 if (dev->data->dev_conf.rxmode.offloads &
1589                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1590                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1591                                         "supported in fm10k");
1592         }
1593
1594         if (mask & ETH_VLAN_FILTER_MASK) {
1595                 if (!(dev->data->dev_conf.rxmode.offloads &
1596                         DEV_RX_OFFLOAD_VLAN_FILTER))
1597                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1598         }
1599
1600         return 0;
1601 }
1602
1603 /* Add/Remove a MAC address, and update filters to main VSI */
1604 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1605                 const u8 *mac, bool add, uint32_t pool)
1606 {
1607         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1608         struct fm10k_macvlan_filter_info *macvlan;
1609         uint32_t i, j, k;
1610
1611         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1612
1613         if (pool != MAIN_VSI_POOL_NUMBER) {
1614                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1615                         "mac to pool %u", pool);
1616                 return;
1617         }
1618         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1619                 if (!macvlan->vfta[j])
1620                         continue;
1621                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1622                         if (!(macvlan->vfta[j] & (1 << k)))
1623                                 continue;
1624                         if (i + 1 > macvlan->vlan_num) {
1625                                 PMD_INIT_LOG(ERR, "vlan number not match");
1626                                 return;
1627                         }
1628                         fm10k_mbx_lock(hw);
1629                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1630                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1631                         fm10k_mbx_unlock(hw);
1632                         i++;
1633                 }
1634         }
1635 }
1636
1637 /* Add/Remove a MAC address, and update filters to VMDQ */
1638 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1639                 const u8 *mac, bool add, uint32_t pool)
1640 {
1641         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1642         struct fm10k_macvlan_filter_info *macvlan;
1643         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1644         uint32_t i;
1645
1646         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1647         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1648
1649         if (pool > macvlan->nb_queue_pools) {
1650                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1651                         " Max pool is %u",
1652                         pool, macvlan->nb_queue_pools);
1653                 return;
1654         }
1655         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1656                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1657                         continue;
1658                 fm10k_mbx_lock(hw);
1659                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1660                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1661                 fm10k_mbx_unlock(hw);
1662         }
1663 }
1664
1665 /* Add/Remove a MAC address, and update filters */
1666 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1667                 const u8 *mac, bool add, uint32_t pool)
1668 {
1669         struct fm10k_macvlan_filter_info *macvlan;
1670
1671         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1672
1673         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1674                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1675         else
1676                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1677
1678         if (add)
1679                 macvlan->mac_num++;
1680         else
1681                 macvlan->mac_num--;
1682 }
1683
1684 /* Add a MAC address, and update filters */
1685 static int
1686 fm10k_macaddr_add(struct rte_eth_dev *dev,
1687                 struct rte_ether_addr *mac_addr,
1688                 uint32_t index,
1689                 uint32_t pool)
1690 {
1691         struct fm10k_macvlan_filter_info *macvlan;
1692
1693         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1694         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1695         macvlan->mac_vmdq_id[index] = pool;
1696         return 0;
1697 }
1698
1699 /* Remove a MAC address, and update filters */
1700 static void
1701 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1702 {
1703         struct rte_eth_dev_data *data = dev->data;
1704         struct fm10k_macvlan_filter_info *macvlan;
1705
1706         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1707         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1708                         FALSE, macvlan->mac_vmdq_id[index]);
1709         macvlan->mac_vmdq_id[index] = 0;
1710 }
1711
1712 static inline int
1713 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1714 {
1715         if ((request < min) || (request > max) || ((request % mult) != 0))
1716                 return -1;
1717         else
1718                 return 0;
1719 }
1720
1721
1722 static inline int
1723 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1724 {
1725         if ((request < min) || (request > max) || ((div % request) != 0))
1726                 return -1;
1727         else
1728                 return 0;
1729 }
1730
1731 static inline int
1732 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1733 {
1734         uint16_t rx_free_thresh;
1735
1736         if (conf->rx_free_thresh == 0)
1737                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1738         else
1739                 rx_free_thresh = conf->rx_free_thresh;
1740
1741         /* make sure the requested threshold satisfies the constraints */
1742         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1743                         FM10K_RX_FREE_THRESH_MAX(q),
1744                         FM10K_RX_FREE_THRESH_DIV(q),
1745                         rx_free_thresh)) {
1746                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1747                         "less than or equal to %u, "
1748                         "greater than or equal to %u, "
1749                         "and a divisor of %u",
1750                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1751                         FM10K_RX_FREE_THRESH_MIN(q),
1752                         FM10K_RX_FREE_THRESH_DIV(q));
1753                 return -EINVAL;
1754         }
1755
1756         q->alloc_thresh = rx_free_thresh;
1757         q->drop_en = conf->rx_drop_en;
1758         q->rx_deferred_start = conf->rx_deferred_start;
1759
1760         return 0;
1761 }
1762
1763 /*
1764  * Hardware requires specific alignment for Rx packet buffers. At
1765  * least one of the following two conditions must be satisfied.
1766  *  1. Address is 512B aligned
1767  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1768  *
1769  * As such, the driver may need to adjust the DMA address within the
1770  * buffer by up to 512B.
1771  *
1772  * return 1 if the element size is valid, otherwise return 0.
1773  */
1774 static int
1775 mempool_element_size_valid(struct rte_mempool *mp)
1776 {
1777         uint32_t min_size;
1778
1779         /* elt_size includes mbuf header and headroom */
1780         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1781                         RTE_PKTMBUF_HEADROOM;
1782
1783         /* account for up to 512B of alignment */
1784         min_size -= FM10K_RX_DATABUF_ALIGN;
1785
1786         /* sanity check for overflow */
1787         if (min_size > mp->elt_size)
1788                 return 0;
1789
1790         /* size is valid */
1791         return 1;
1792 }
1793
1794 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1795 {
1796         RTE_SET_USED(dev);
1797
1798         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1799 }
1800
1801 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1802 {
1803         RTE_SET_USED(dev);
1804
1805         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1806                            DEV_RX_OFFLOAD_VLAN_FILTER |
1807                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1808                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1809                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1810                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1811                            DEV_RX_OFFLOAD_HEADER_SPLIT |
1812                            DEV_RX_OFFLOAD_RSS_HASH);
1813 }
1814
1815 static int
1816 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1817         uint16_t nb_desc, unsigned int socket_id,
1818         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1819 {
1820         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1821         struct fm10k_dev_info *dev_info =
1822                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1823         struct fm10k_rx_queue *q;
1824         const struct rte_memzone *mz;
1825         uint64_t offloads;
1826
1827         PMD_INIT_FUNC_TRACE();
1828
1829         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1830
1831         /* make sure the mempool element size can account for alignment. */
1832         if (!mempool_element_size_valid(mp)) {
1833                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1834                 return -EINVAL;
1835         }
1836
1837         /* make sure a valid number of descriptors have been requested */
1838         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1839                                 FM10K_MULT_RX_DESC, nb_desc)) {
1840                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1841                         "less than or equal to %"PRIu32", "
1842                         "greater than or equal to %u, "
1843                         "and a multiple of %u",
1844                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1845                         FM10K_MULT_RX_DESC);
1846                 return -EINVAL;
1847         }
1848
1849         /*
1850          * if this queue existed already, free the associated memory. The
1851          * queue cannot be reused in case we need to allocate memory on
1852          * different socket than was previously used.
1853          */
1854         if (dev->data->rx_queues[queue_id] != NULL) {
1855                 rx_queue_free(dev->data->rx_queues[queue_id]);
1856                 dev->data->rx_queues[queue_id] = NULL;
1857         }
1858
1859         /* allocate memory for the queue structure */
1860         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1861                                 socket_id);
1862         if (q == NULL) {
1863                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1864                 return -ENOMEM;
1865         }
1866
1867         /* setup queue */
1868         q->mp = mp;
1869         q->nb_desc = nb_desc;
1870         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1871         q->port_id = dev->data->port_id;
1872         q->queue_id = queue_id;
1873         q->tail_ptr = (volatile uint32_t *)
1874                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1875         q->offloads = offloads;
1876         if (handle_rxconf(q, conf))
1877                 return -EINVAL;
1878
1879         /* allocate memory for the software ring */
1880         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1881                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1882                         RTE_CACHE_LINE_SIZE, socket_id);
1883         if (q->sw_ring == NULL) {
1884                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1885                 rte_free(q);
1886                 return -ENOMEM;
1887         }
1888
1889         /*
1890          * allocate memory for the hardware descriptor ring. A memzone large
1891          * enough to hold the maximum ring size is requested to allow for
1892          * resizing in later calls to the queue setup function.
1893          */
1894         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1895                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1896                                       socket_id);
1897         if (mz == NULL) {
1898                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1899                 rte_free(q->sw_ring);
1900                 rte_free(q);
1901                 return -ENOMEM;
1902         }
1903         q->hw_ring = mz->addr;
1904         q->hw_ring_phys_addr = mz->iova;
1905
1906         /* Check if number of descs satisfied Vector requirement */
1907         if (!rte_is_power_of_2(nb_desc)) {
1908                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1909                                     "preconditions - canceling the feature for "
1910                                     "the whole port[%d]",
1911                              q->queue_id, q->port_id);
1912                 dev_info->rx_vec_allowed = false;
1913         } else
1914                 fm10k_rxq_vec_setup(q);
1915
1916         dev->data->rx_queues[queue_id] = q;
1917         return 0;
1918 }
1919
1920 static void
1921 fm10k_rx_queue_release(void *queue)
1922 {
1923         PMD_INIT_FUNC_TRACE();
1924
1925         rx_queue_free(queue);
1926 }
1927
1928 static inline int
1929 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1930 {
1931         uint16_t tx_free_thresh;
1932         uint16_t tx_rs_thresh;
1933
1934         /* constraint MACROs require that tx_free_thresh is configured
1935          * before tx_rs_thresh */
1936         if (conf->tx_free_thresh == 0)
1937                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1938         else
1939                 tx_free_thresh = conf->tx_free_thresh;
1940
1941         /* make sure the requested threshold satisfies the constraints */
1942         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1943                         FM10K_TX_FREE_THRESH_MAX(q),
1944                         FM10K_TX_FREE_THRESH_DIV(q),
1945                         tx_free_thresh)) {
1946                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1947                         "less than or equal to %u, "
1948                         "greater than or equal to %u, "
1949                         "and a divisor of %u",
1950                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1951                         FM10K_TX_FREE_THRESH_MIN(q),
1952                         FM10K_TX_FREE_THRESH_DIV(q));
1953                 return -EINVAL;
1954         }
1955
1956         q->free_thresh = tx_free_thresh;
1957
1958         if (conf->tx_rs_thresh == 0)
1959                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1960         else
1961                 tx_rs_thresh = conf->tx_rs_thresh;
1962
1963         q->tx_deferred_start = conf->tx_deferred_start;
1964
1965         /* make sure the requested threshold satisfies the constraints */
1966         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1967                         FM10K_TX_RS_THRESH_MAX(q),
1968                         FM10K_TX_RS_THRESH_DIV(q),
1969                         tx_rs_thresh)) {
1970                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1971                         "less than or equal to %u, "
1972                         "greater than or equal to %u, "
1973                         "and a divisor of %u",
1974                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1975                         FM10K_TX_RS_THRESH_MIN(q),
1976                         FM10K_TX_RS_THRESH_DIV(q));
1977                 return -EINVAL;
1978         }
1979
1980         q->rs_thresh = tx_rs_thresh;
1981
1982         return 0;
1983 }
1984
1985 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1986 {
1987         RTE_SET_USED(dev);
1988
1989         return 0;
1990 }
1991
1992 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1993 {
1994         RTE_SET_USED(dev);
1995
1996         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1997                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1998                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1999                           DEV_TX_OFFLOAD_UDP_CKSUM   |
2000                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2001                           DEV_TX_OFFLOAD_TCP_TSO);
2002 }
2003
2004 static int
2005 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2006         uint16_t nb_desc, unsigned int socket_id,
2007         const struct rte_eth_txconf *conf)
2008 {
2009         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2010         struct fm10k_tx_queue *q;
2011         const struct rte_memzone *mz;
2012         uint64_t offloads;
2013
2014         PMD_INIT_FUNC_TRACE();
2015
2016         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2017
2018         /* make sure a valid number of descriptors have been requested */
2019         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2020                                 FM10K_MULT_TX_DESC, nb_desc)) {
2021                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2022                         "less than or equal to %"PRIu32", "
2023                         "greater than or equal to %u, "
2024                         "and a multiple of %u",
2025                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2026                         FM10K_MULT_TX_DESC);
2027                 return -EINVAL;
2028         }
2029
2030         /*
2031          * if this queue existed already, free the associated memory. The
2032          * queue cannot be reused in case we need to allocate memory on
2033          * different socket than was previously used.
2034          */
2035         if (dev->data->tx_queues[queue_id] != NULL) {
2036                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2037
2038                 tx_queue_free(txq);
2039                 dev->data->tx_queues[queue_id] = NULL;
2040         }
2041
2042         /* allocate memory for the queue structure */
2043         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2044                                 socket_id);
2045         if (q == NULL) {
2046                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2047                 return -ENOMEM;
2048         }
2049
2050         /* setup queue */
2051         q->nb_desc = nb_desc;
2052         q->port_id = dev->data->port_id;
2053         q->queue_id = queue_id;
2054         q->offloads = offloads;
2055         q->ops = &def_txq_ops;
2056         q->tail_ptr = (volatile uint32_t *)
2057                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2058         if (handle_txconf(q, conf))
2059                 return -EINVAL;
2060
2061         /* allocate memory for the software ring */
2062         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2063                                         nb_desc * sizeof(struct rte_mbuf *),
2064                                         RTE_CACHE_LINE_SIZE, socket_id);
2065         if (q->sw_ring == NULL) {
2066                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2067                 rte_free(q);
2068                 return -ENOMEM;
2069         }
2070
2071         /*
2072          * allocate memory for the hardware descriptor ring. A memzone large
2073          * enough to hold the maximum ring size is requested to allow for
2074          * resizing in later calls to the queue setup function.
2075          */
2076         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2077                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2078                                       socket_id);
2079         if (mz == NULL) {
2080                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2081                 rte_free(q->sw_ring);
2082                 rte_free(q);
2083                 return -ENOMEM;
2084         }
2085         q->hw_ring = mz->addr;
2086         q->hw_ring_phys_addr = mz->iova;
2087
2088         /*
2089          * allocate memory for the RS bit tracker. Enough slots to hold the
2090          * descriptor index for each RS bit needing to be set are required.
2091          */
2092         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2093                                 ((nb_desc + 1) / q->rs_thresh) *
2094                                 sizeof(uint16_t),
2095                                 RTE_CACHE_LINE_SIZE, socket_id);
2096         if (q->rs_tracker.list == NULL) {
2097                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2098                 rte_free(q->sw_ring);
2099                 rte_free(q);
2100                 return -ENOMEM;
2101         }
2102
2103         dev->data->tx_queues[queue_id] = q;
2104         return 0;
2105 }
2106
2107 static void
2108 fm10k_tx_queue_release(void *queue)
2109 {
2110         struct fm10k_tx_queue *q = queue;
2111         PMD_INIT_FUNC_TRACE();
2112
2113         tx_queue_free(q);
2114 }
2115
2116 static int
2117 fm10k_reta_update(struct rte_eth_dev *dev,
2118                         struct rte_eth_rss_reta_entry64 *reta_conf,
2119                         uint16_t reta_size)
2120 {
2121         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2122         uint16_t i, j, idx, shift;
2123         uint8_t mask;
2124         uint32_t reta;
2125
2126         PMD_INIT_FUNC_TRACE();
2127
2128         if (reta_size > FM10K_MAX_RSS_INDICES) {
2129                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2130                         "(%d) doesn't match the number hardware can supported "
2131                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2132                 return -EINVAL;
2133         }
2134
2135         /*
2136          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2137          * 128-entries in 32 registers
2138          */
2139         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2140                 idx = i / RTE_RETA_GROUP_SIZE;
2141                 shift = i % RTE_RETA_GROUP_SIZE;
2142                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2143                                 BIT_MASK_PER_UINT32);
2144                 if (mask == 0)
2145                         continue;
2146
2147                 reta = 0;
2148                 if (mask != BIT_MASK_PER_UINT32)
2149                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2150
2151                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2152                         if (mask & (0x1 << j)) {
2153                                 if (mask != 0xF)
2154                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2155                                 reta |= reta_conf[idx].reta[shift + j] <<
2156                                                 (CHAR_BIT * j);
2157                         }
2158                 }
2159                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2160         }
2161
2162         return 0;
2163 }
2164
2165 static int
2166 fm10k_reta_query(struct rte_eth_dev *dev,
2167                         struct rte_eth_rss_reta_entry64 *reta_conf,
2168                         uint16_t reta_size)
2169 {
2170         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2171         uint16_t i, j, idx, shift;
2172         uint8_t mask;
2173         uint32_t reta;
2174
2175         PMD_INIT_FUNC_TRACE();
2176
2177         if (reta_size < FM10K_MAX_RSS_INDICES) {
2178                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2179                         "(%d) doesn't match the number hardware can supported "
2180                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2181                 return -EINVAL;
2182         }
2183
2184         /*
2185          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2186          * 128-entries in 32 registers
2187          */
2188         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2189                 idx = i / RTE_RETA_GROUP_SIZE;
2190                 shift = i % RTE_RETA_GROUP_SIZE;
2191                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2192                                 BIT_MASK_PER_UINT32);
2193                 if (mask == 0)
2194                         continue;
2195
2196                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2197                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2198                         if (mask & (0x1 << j))
2199                                 reta_conf[idx].reta[shift + j] = ((reta >>
2200                                         CHAR_BIT * j) & UINT8_MAX);
2201                 }
2202         }
2203
2204         return 0;
2205 }
2206
2207 static int
2208 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2209         struct rte_eth_rss_conf *rss_conf)
2210 {
2211         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2212         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2213         uint32_t mrqc;
2214         uint64_t hf = rss_conf->rss_hf;
2215         int i;
2216
2217         PMD_INIT_FUNC_TRACE();
2218
2219         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2220                                 FM10K_RSSRK_ENTRIES_PER_REG))
2221                 return -EINVAL;
2222
2223         if (hf == 0)
2224                 return -EINVAL;
2225
2226         mrqc = 0;
2227         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2228         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2229         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2230         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2231         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2232         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2233         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2234         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2235         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2236
2237         /* If the mapping doesn't fit any supported, return */
2238         if (mrqc == 0)
2239                 return -EINVAL;
2240
2241         if (key != NULL)
2242                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2243                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2244
2245         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2246
2247         return 0;
2248 }
2249
2250 static int
2251 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2252         struct rte_eth_rss_conf *rss_conf)
2253 {
2254         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2255         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2256         uint32_t mrqc;
2257         uint64_t hf;
2258         int i;
2259
2260         PMD_INIT_FUNC_TRACE();
2261
2262         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2263                                 FM10K_RSSRK_ENTRIES_PER_REG))
2264                 return -EINVAL;
2265
2266         if (key != NULL)
2267                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2268                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2269
2270         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2271         hf = 0;
2272         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2273         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2274         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2275         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2276         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2277         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2278         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2279         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2280         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2281
2282         rss_conf->rss_hf = hf;
2283
2284         return 0;
2285 }
2286
2287 static void
2288 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2289 {
2290         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2291         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2292
2293         /* Bind all local non-queue interrupt to vector 0 */
2294         int_map |= FM10K_MISC_VEC_ID;
2295
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2298         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2299         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2300         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2301         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2302
2303         /* Enable misc causes */
2304         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2305                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2306                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2307                                 FM10K_EIMR_ENABLE(MAILBOX) |
2308                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2309                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2310                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2311                                 FM10K_EIMR_ENABLE(VFLR));
2312
2313         /* Enable ITR 0 */
2314         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2315                                         FM10K_ITR_MASK_CLEAR);
2316         FM10K_WRITE_FLUSH(hw);
2317 }
2318
2319 static void
2320 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2321 {
2322         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2323         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2324
2325         int_map |= FM10K_MISC_VEC_ID;
2326
2327         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2328         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2329         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2330         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2331         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2332         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2333
2334         /* Disable misc causes */
2335         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2336                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2337                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2338                                 FM10K_EIMR_DISABLE(MAILBOX) |
2339                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2340                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2341                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2342                                 FM10K_EIMR_DISABLE(VFLR));
2343
2344         /* Disable ITR 0 */
2345         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2346         FM10K_WRITE_FLUSH(hw);
2347 }
2348
2349 static void
2350 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2351 {
2352         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2353         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2354
2355         /* Bind all local non-queue interrupt to vector 0 */
2356         int_map |= FM10K_MISC_VEC_ID;
2357
2358         /* Only INT 0 available, other 15 are reserved. */
2359         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2360
2361         /* Enable ITR 0 */
2362         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2363                                         FM10K_ITR_MASK_CLEAR);
2364         FM10K_WRITE_FLUSH(hw);
2365 }
2366
2367 static void
2368 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2369 {
2370         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2371         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2372
2373         int_map |= FM10K_MISC_VEC_ID;
2374
2375         /* Only INT 0 available, other 15 are reserved. */
2376         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2377
2378         /* Disable ITR 0 */
2379         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2380         FM10K_WRITE_FLUSH(hw);
2381 }
2382
2383 static int
2384 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2385 {
2386         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2387         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2388
2389         /* Enable ITR */
2390         if (hw->mac.type == fm10k_mac_pf)
2391                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2392                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2393         else
2394                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2395                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2396         rte_intr_ack(&pdev->intr_handle);
2397         return 0;
2398 }
2399
2400 static int
2401 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2402 {
2403         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2404         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2405
2406         /* Disable ITR */
2407         if (hw->mac.type == fm10k_mac_pf)
2408                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2409                         FM10K_ITR_MASK_SET);
2410         else
2411                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2412                         FM10K_ITR_MASK_SET);
2413         return 0;
2414 }
2415
2416 static int
2417 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2418 {
2419         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2420         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2421         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2422         uint32_t intr_vector, vec;
2423         uint16_t queue_id;
2424         int result = 0;
2425
2426         /* fm10k needs one separate interrupt for mailbox,
2427          * so only drivers which support multiple interrupt vectors
2428          * e.g. vfio-pci can work for fm10k interrupt mode
2429          */
2430         if (!rte_intr_cap_multiple(intr_handle) ||
2431                         dev->data->dev_conf.intr_conf.rxq == 0)
2432                 return result;
2433
2434         intr_vector = dev->data->nb_rx_queues;
2435
2436         /* disable interrupt first */
2437         rte_intr_disable(intr_handle);
2438         if (hw->mac.type == fm10k_mac_pf)
2439                 fm10k_dev_disable_intr_pf(dev);
2440         else
2441                 fm10k_dev_disable_intr_vf(dev);
2442
2443         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2444                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2445                 result = -EIO;
2446         }
2447
2448         if (rte_intr_dp_is_en(intr_handle) && !result) {
2449                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2450                         dev->data->nb_rx_queues * sizeof(int), 0);
2451                 if (intr_handle->intr_vec) {
2452                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2453                                         queue_id < dev->data->nb_rx_queues;
2454                                         queue_id++) {
2455                                 intr_handle->intr_vec[queue_id] = vec;
2456                                 if (vec < intr_handle->nb_efd - 1
2457                                                 + FM10K_RX_VEC_START)
2458                                         vec++;
2459                         }
2460                 } else {
2461                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2462                                 " intr_vec", dev->data->nb_rx_queues);
2463                         rte_intr_efd_disable(intr_handle);
2464                         result = -ENOMEM;
2465                 }
2466         }
2467
2468         if (hw->mac.type == fm10k_mac_pf)
2469                 fm10k_dev_enable_intr_pf(dev);
2470         else
2471                 fm10k_dev_enable_intr_vf(dev);
2472         rte_intr_enable(intr_handle);
2473         hw->mac.ops.update_int_moderator(hw);
2474         return result;
2475 }
2476
2477 static int
2478 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2479 {
2480         struct fm10k_fault fault;
2481         int err;
2482         const char *estr = "Unknown error";
2483
2484         /* Process PCA fault */
2485         if (eicr & FM10K_EICR_PCA_FAULT) {
2486                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2487                 if (err)
2488                         goto error;
2489                 switch (fault.type) {
2490                 case PCA_NO_FAULT:
2491                         estr = "PCA_NO_FAULT"; break;
2492                 case PCA_UNMAPPED_ADDR:
2493                         estr = "PCA_UNMAPPED_ADDR"; break;
2494                 case PCA_BAD_QACCESS_PF:
2495                         estr = "PCA_BAD_QACCESS_PF"; break;
2496                 case PCA_BAD_QACCESS_VF:
2497                         estr = "PCA_BAD_QACCESS_VF"; break;
2498                 case PCA_MALICIOUS_REQ:
2499                         estr = "PCA_MALICIOUS_REQ"; break;
2500                 case PCA_POISONED_TLP:
2501                         estr = "PCA_POISONED_TLP"; break;
2502                 case PCA_TLP_ABORT:
2503                         estr = "PCA_TLP_ABORT"; break;
2504                 default:
2505                         goto error;
2506                 }
2507                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2508                         estr, fault.func ? "VF" : "PF", fault.func,
2509                         fault.address, fault.specinfo);
2510         }
2511
2512         /* Process THI fault */
2513         if (eicr & FM10K_EICR_THI_FAULT) {
2514                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2515                 if (err)
2516                         goto error;
2517                 switch (fault.type) {
2518                 case THI_NO_FAULT:
2519                         estr = "THI_NO_FAULT"; break;
2520                 case THI_MAL_DIS_Q_FAULT:
2521                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2522                 default:
2523                         goto error;
2524                 }
2525                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2526                         estr, fault.func ? "VF" : "PF", fault.func,
2527                         fault.address, fault.specinfo);
2528         }
2529
2530         /* Process FUM fault */
2531         if (eicr & FM10K_EICR_FUM_FAULT) {
2532                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2533                 if (err)
2534                         goto error;
2535                 switch (fault.type) {
2536                 case FUM_NO_FAULT:
2537                         estr = "FUM_NO_FAULT"; break;
2538                 case FUM_UNMAPPED_ADDR:
2539                         estr = "FUM_UNMAPPED_ADDR"; break;
2540                 case FUM_POISONED_TLP:
2541                         estr = "FUM_POISONED_TLP"; break;
2542                 case FUM_BAD_VF_QACCESS:
2543                         estr = "FUM_BAD_VF_QACCESS"; break;
2544                 case FUM_ADD_DECODE_ERR:
2545                         estr = "FUM_ADD_DECODE_ERR"; break;
2546                 case FUM_RO_ERROR:
2547                         estr = "FUM_RO_ERROR"; break;
2548                 case FUM_QPRC_CRC_ERROR:
2549                         estr = "FUM_QPRC_CRC_ERROR"; break;
2550                 case FUM_CSR_TIMEOUT:
2551                         estr = "FUM_CSR_TIMEOUT"; break;
2552                 case FUM_INVALID_TYPE:
2553                         estr = "FUM_INVALID_TYPE"; break;
2554                 case FUM_INVALID_LENGTH:
2555                         estr = "FUM_INVALID_LENGTH"; break;
2556                 case FUM_INVALID_BE:
2557                         estr = "FUM_INVALID_BE"; break;
2558                 case FUM_INVALID_ALIGN:
2559                         estr = "FUM_INVALID_ALIGN"; break;
2560                 default:
2561                         goto error;
2562                 }
2563                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2564                         estr, fault.func ? "VF" : "PF", fault.func,
2565                         fault.address, fault.specinfo);
2566         }
2567
2568         return 0;
2569 error:
2570         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2571         return err;
2572 }
2573
2574 /**
2575  * PF interrupt handler triggered by NIC for handling specific interrupt.
2576  *
2577  * @param handle
2578  *  Pointer to interrupt handle.
2579  * @param param
2580  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2581  *
2582  * @return
2583  *  void
2584  */
2585 static void
2586 fm10k_dev_interrupt_handler_pf(void *param)
2587 {
2588         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2589         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2590         uint32_t cause, status;
2591         struct fm10k_dev_info *dev_info =
2592                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2593         int status_mbx;
2594         s32 err;
2595
2596         if (hw->mac.type != fm10k_mac_pf)
2597                 return;
2598
2599         cause = FM10K_READ_REG(hw, FM10K_EICR);
2600
2601         /* Handle PCI fault cases */
2602         if (cause & FM10K_EICR_FAULT_MASK) {
2603                 PMD_INIT_LOG(ERR, "INT: find fault!");
2604                 fm10k_dev_handle_fault(hw, cause);
2605         }
2606
2607         /* Handle switch up/down */
2608         if (cause & FM10K_EICR_SWITCHNOTREADY)
2609                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2610
2611         if (cause & FM10K_EICR_SWITCHREADY) {
2612                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2613                 if (dev_info->sm_down == 1) {
2614                         fm10k_mbx_lock(hw);
2615
2616                         /* For recreating logical ports */
2617                         status_mbx = hw->mac.ops.update_lport_state(hw,
2618                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2619                         if (status_mbx == FM10K_SUCCESS)
2620                                 PMD_INIT_LOG(INFO,
2621                                         "INT: Recreated Logical port");
2622                         else
2623                                 PMD_INIT_LOG(INFO,
2624                                         "INT: Logical ports weren't recreated");
2625
2626                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2627                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2628                         if (status_mbx != FM10K_SUCCESS)
2629                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2630
2631                         fm10k_mbx_unlock(hw);
2632
2633                         /* first clear the internal SW recording structure */
2634                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2635                                                 ETH_MQ_RX_VMDQ_FLAG))
2636                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2637                                         false);
2638
2639                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2640                                         MAIN_VSI_POOL_NUMBER);
2641
2642                         /*
2643                          * Add default mac address and vlan for the logical
2644                          * ports that have been created, leave to the
2645                          * application to fully recover Rx filtering.
2646                          */
2647                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2648                                         MAIN_VSI_POOL_NUMBER);
2649
2650                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2651                                                 ETH_MQ_RX_VMDQ_FLAG))
2652                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2653                                         true);
2654
2655                         dev_info->sm_down = 0;
2656                         _rte_eth_dev_callback_process(dev,
2657                                         RTE_ETH_EVENT_INTR_LSC,
2658                                         NULL);
2659                 }
2660         }
2661
2662         /* Handle mailbox message */
2663         fm10k_mbx_lock(hw);
2664         err = hw->mbx.ops.process(hw, &hw->mbx);
2665         fm10k_mbx_unlock(hw);
2666
2667         if (err == FM10K_ERR_RESET_REQUESTED) {
2668                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2669                 dev_info->sm_down = 1;
2670                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2671                                 NULL);
2672         }
2673
2674         /* Handle SRAM error */
2675         if (cause & FM10K_EICR_SRAMERROR) {
2676                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2677
2678                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2679                 /* Write to clear pending bits */
2680                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2681
2682                 /* Todo: print out error message after shared code  updates */
2683         }
2684
2685         /* Clear these 3 events if having any */
2686         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2687                  FM10K_EICR_SWITCHREADY;
2688         if (cause)
2689                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2690
2691         /* Re-enable interrupt from device side */
2692         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2693                                         FM10K_ITR_MASK_CLEAR);
2694         /* Re-enable interrupt from host side */
2695         rte_intr_ack(dev->intr_handle);
2696 }
2697
2698 /**
2699  * VF interrupt handler triggered by NIC for handling specific interrupt.
2700  *
2701  * @param handle
2702  *  Pointer to interrupt handle.
2703  * @param param
2704  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2705  *
2706  * @return
2707  *  void
2708  */
2709 static void
2710 fm10k_dev_interrupt_handler_vf(void *param)
2711 {
2712         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2713         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2714         struct fm10k_mbx_info *mbx = &hw->mbx;
2715         struct fm10k_dev_info *dev_info =
2716                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2717         const enum fm10k_mbx_state state = mbx->state;
2718         int status_mbx;
2719
2720         if (hw->mac.type != fm10k_mac_vf)
2721                 return;
2722
2723         /* Handle mailbox message if lock is acquired */
2724         fm10k_mbx_lock(hw);
2725         hw->mbx.ops.process(hw, &hw->mbx);
2726         fm10k_mbx_unlock(hw);
2727
2728         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2729                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2730
2731                 fm10k_mbx_lock(hw);
2732                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2733                                 MAX_LPORT_NUM, 1);
2734                 fm10k_mbx_unlock(hw);
2735
2736                 /* Setting reset flag */
2737                 dev_info->sm_down = 1;
2738                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2739                                 NULL);
2740         }
2741
2742         if (dev_info->sm_down == 1 &&
2743                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2744                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2745                 fm10k_mbx_lock(hw);
2746                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2747                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2748                 if (status_mbx != FM10K_SUCCESS)
2749                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2750                 fm10k_mbx_unlock(hw);
2751
2752                 /* first clear the internal SW recording structure */
2753                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2754                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2755                                 MAIN_VSI_POOL_NUMBER);
2756
2757                 /*
2758                  * Add default mac address and vlan for the logical ports that
2759                  * have been created, leave to the application to fully recover
2760                  * Rx filtering.
2761                  */
2762                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2763                                 MAIN_VSI_POOL_NUMBER);
2764                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2765
2766                 dev_info->sm_down = 0;
2767                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2768                                 NULL);
2769         }
2770
2771         /* Re-enable interrupt from device side */
2772         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2773                                         FM10K_ITR_MASK_CLEAR);
2774         /* Re-enable interrupt from host side */
2775         rte_intr_ack(dev->intr_handle);
2776 }
2777
2778 /* Mailbox message handler in VF */
2779 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2780         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2781         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2782         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2783         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2784 };
2785
2786 static int
2787 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2788 {
2789         int err = 0;
2790
2791         /* Initialize mailbox lock */
2792         fm10k_mbx_initlock(hw);
2793
2794         /* Replace default message handler with new ones */
2795         if (hw->mac.type == fm10k_mac_vf)
2796                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2797
2798         if (err) {
2799                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2800                                 err);
2801                 return err;
2802         }
2803         /* Connect to SM for PF device or PF for VF device */
2804         return hw->mbx.ops.connect(hw, &hw->mbx);
2805 }
2806
2807 static void
2808 fm10k_close_mbx_service(struct fm10k_hw *hw)
2809 {
2810         /* Disconnect from SM for PF device or PF for VF device */
2811         hw->mbx.ops.disconnect(hw, &hw->mbx);
2812 }
2813
2814 static void
2815 fm10k_dev_close(struct rte_eth_dev *dev)
2816 {
2817         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2818         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2819         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2820
2821         PMD_INIT_FUNC_TRACE();
2822
2823         fm10k_mbx_lock(hw);
2824         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2825                 MAX_LPORT_NUM, false);
2826         fm10k_mbx_unlock(hw);
2827
2828         /* allow 100ms for device to quiesce */
2829         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2830
2831         /* Stop mailbox service first */
2832         fm10k_close_mbx_service(hw);
2833         fm10k_dev_stop(dev);
2834         fm10k_dev_queue_release(dev);
2835         fm10k_stop_hw(hw);
2836
2837         dev->dev_ops = NULL;
2838         dev->rx_pkt_burst = NULL;
2839         dev->tx_pkt_burst = NULL;
2840
2841         /* disable uio/vfio intr */
2842         rte_intr_disable(intr_handle);
2843
2844         /*PF/VF has different interrupt handling mechanism */
2845         if (hw->mac.type == fm10k_mac_pf) {
2846                 /* disable interrupt */
2847                 fm10k_dev_disable_intr_pf(dev);
2848
2849                 /* unregister callback func to eal lib */
2850                 rte_intr_callback_unregister(intr_handle,
2851                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2852         } else {
2853                 /* disable interrupt */
2854                 fm10k_dev_disable_intr_vf(dev);
2855
2856                 rte_intr_callback_unregister(intr_handle,
2857                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2858         }
2859 }
2860
2861 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2862         .dev_configure          = fm10k_dev_configure,
2863         .dev_start              = fm10k_dev_start,
2864         .dev_stop               = fm10k_dev_stop,
2865         .dev_close              = fm10k_dev_close,
2866         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2867         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2868         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2869         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2870         .stats_get              = fm10k_stats_get,
2871         .xstats_get             = fm10k_xstats_get,
2872         .xstats_get_names       = fm10k_xstats_get_names,
2873         .stats_reset            = fm10k_stats_reset,
2874         .xstats_reset           = fm10k_stats_reset,
2875         .link_update            = fm10k_link_update,
2876         .dev_infos_get          = fm10k_dev_infos_get,
2877         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2878         .vlan_filter_set        = fm10k_vlan_filter_set,
2879         .vlan_offload_set       = fm10k_vlan_offload_set,
2880         .mac_addr_add           = fm10k_macaddr_add,
2881         .mac_addr_remove        = fm10k_macaddr_remove,
2882         .rx_queue_start         = fm10k_dev_rx_queue_start,
2883         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2884         .tx_queue_start         = fm10k_dev_tx_queue_start,
2885         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2886         .rx_queue_setup         = fm10k_rx_queue_setup,
2887         .rx_queue_release       = fm10k_rx_queue_release,
2888         .tx_queue_setup         = fm10k_tx_queue_setup,
2889         .tx_queue_release       = fm10k_tx_queue_release,
2890         .rx_queue_count         = fm10k_dev_rx_queue_count,
2891         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2892         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2893         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2894         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2895         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2896         .reta_update            = fm10k_reta_update,
2897         .reta_query             = fm10k_reta_query,
2898         .rss_hash_update        = fm10k_rss_hash_update,
2899         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2900 };
2901
2902 static int ftag_check_handler(__rte_unused const char *key,
2903                 const char *value, __rte_unused void *opaque)
2904 {
2905         if (strcmp(value, "1"))
2906                 return -1;
2907
2908         return 0;
2909 }
2910
2911 static int
2912 fm10k_check_ftag(struct rte_devargs *devargs)
2913 {
2914         struct rte_kvargs *kvlist;
2915         const char *ftag_key = "enable_ftag";
2916
2917         if (devargs == NULL)
2918                 return 0;
2919
2920         kvlist = rte_kvargs_parse(devargs->args, NULL);
2921         if (kvlist == NULL)
2922                 return 0;
2923
2924         if (!rte_kvargs_count(kvlist, ftag_key)) {
2925                 rte_kvargs_free(kvlist);
2926                 return 0;
2927         }
2928         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2929         if (rte_kvargs_process(kvlist, ftag_key,
2930                                 ftag_check_handler, NULL) < 0) {
2931                 rte_kvargs_free(kvlist);
2932                 return 0;
2933         }
2934         rte_kvargs_free(kvlist);
2935
2936         return 1;
2937 }
2938
2939 static uint16_t
2940 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2941                     uint16_t nb_pkts)
2942 {
2943         uint16_t nb_tx = 0;
2944         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2945
2946         while (nb_pkts) {
2947                 uint16_t ret, num;
2948
2949                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2950                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2951                                                  num);
2952                 nb_tx += ret;
2953                 nb_pkts -= ret;
2954                 if (ret < num)
2955                         break;
2956         }
2957
2958         return nb_tx;
2959 }
2960
2961 static void __rte_cold
2962 fm10k_set_tx_function(struct rte_eth_dev *dev)
2963 {
2964         struct fm10k_tx_queue *txq;
2965         int i;
2966         int use_sse = 1;
2967         uint16_t tx_ftag_en = 0;
2968
2969         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2970                 /* primary process has set the ftag flag and offloads */
2971                 txq = dev->data->tx_queues[0];
2972                 if (fm10k_tx_vec_condition_check(txq)) {
2973                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2974                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2975                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2976                 } else {
2977                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2978                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2979                         dev->tx_pkt_prepare = NULL;
2980                 }
2981                 return;
2982         }
2983
2984         if (fm10k_check_ftag(dev->device->devargs))
2985                 tx_ftag_en = 1;
2986
2987         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2988                 txq = dev->data->tx_queues[i];
2989                 txq->tx_ftag_en = tx_ftag_en;
2990                 /* Check if Vector Tx is satisfied */
2991                 if (fm10k_tx_vec_condition_check(txq))
2992                         use_sse = 0;
2993         }
2994
2995         if (use_sse) {
2996                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2997                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2998                         txq = dev->data->tx_queues[i];
2999                         fm10k_txq_vec_setup(txq);
3000                 }
3001                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
3002                 dev->tx_pkt_prepare = NULL;
3003         } else {
3004                 dev->tx_pkt_burst = fm10k_xmit_pkts;
3005                 dev->tx_pkt_prepare = fm10k_prep_pkts;
3006                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
3007         }
3008 }
3009
3010 static void __rte_cold
3011 fm10k_set_rx_function(struct rte_eth_dev *dev)
3012 {
3013         struct fm10k_dev_info *dev_info =
3014                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3015         uint16_t i, rx_using_sse;
3016         uint16_t rx_ftag_en = 0;
3017
3018         if (fm10k_check_ftag(dev->device->devargs))
3019                 rx_ftag_en = 1;
3020
3021         /* In order to allow Vector Rx there are a few configuration
3022          * conditions to be met.
3023          */
3024         if (!fm10k_rx_vec_condition_check(dev) &&
3025                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3026                 if (dev->data->scattered_rx)
3027                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3028                 else
3029                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3030         } else if (dev->data->scattered_rx)
3031                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3032         else
3033                 dev->rx_pkt_burst = fm10k_recv_pkts;
3034
3035         rx_using_sse =
3036                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3037                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3038
3039         if (rx_using_sse)
3040                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3041         else
3042                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3043
3044         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3045                 return;
3046
3047         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3048                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3049
3050                 rxq->rx_using_sse = rx_using_sse;
3051                 rxq->rx_ftag_en = rx_ftag_en;
3052         }
3053 }
3054
3055 static void
3056 fm10k_params_init(struct rte_eth_dev *dev)
3057 {
3058         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3059         struct fm10k_dev_info *info =
3060                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3061
3062         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3063          * there is no way to get link status without reading BAR4.  Until this
3064          * works, assume we have maximum bandwidth.
3065          * @todo - fix bus info
3066          */
3067         hw->bus_caps.speed = fm10k_bus_speed_8000;
3068         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3069         hw->bus_caps.payload = fm10k_bus_payload_512;
3070         hw->bus.speed = fm10k_bus_speed_8000;
3071         hw->bus.width = fm10k_bus_width_pcie_x8;
3072         hw->bus.payload = fm10k_bus_payload_256;
3073
3074         info->rx_vec_allowed = true;
3075         info->sm_down = false;
3076 }
3077
3078 static int
3079 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3080 {
3081         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3082         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3083         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3084         int diag, i;
3085         struct fm10k_macvlan_filter_info *macvlan;
3086
3087         PMD_INIT_FUNC_TRACE();
3088
3089         dev->dev_ops = &fm10k_eth_dev_ops;
3090         dev->rx_pkt_burst = &fm10k_recv_pkts;
3091         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3092         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3093
3094         /*
3095          * Primary process does the whole initialization, for secondary
3096          * processes, we just select the same Rx and Tx function as primary.
3097          */
3098         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3099                 fm10k_set_rx_function(dev);
3100                 fm10k_set_tx_function(dev);
3101                 return 0;
3102         }
3103
3104         rte_eth_copy_pci_info(dev, pdev);
3105
3106         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3107         memset(macvlan, 0, sizeof(*macvlan));
3108         /* Vendor and Device ID need to be set before init of shared code */
3109         memset(hw, 0, sizeof(*hw));
3110         hw->device_id = pdev->id.device_id;
3111         hw->vendor_id = pdev->id.vendor_id;
3112         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3113         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3114         hw->revision_id = 0;
3115         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3116         if (hw->hw_addr == NULL) {
3117                 PMD_INIT_LOG(ERR, "Bad mem resource."
3118                         " Try to blacklist unused devices.");
3119                 return -EIO;
3120         }
3121
3122         /* Store fm10k_adapter pointer */
3123         hw->back = dev->data->dev_private;
3124
3125         /* Initialize the shared code */
3126         diag = fm10k_init_shared_code(hw);
3127         if (diag != FM10K_SUCCESS) {
3128                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3129                 return -EIO;
3130         }
3131
3132         /* Initialize parameters */
3133         fm10k_params_init(dev);
3134
3135         /* Initialize the hw */
3136         diag = fm10k_init_hw(hw);
3137         if (diag != FM10K_SUCCESS) {
3138                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3139                 return -EIO;
3140         }
3141
3142         /* Initialize MAC address(es) */
3143         dev->data->mac_addrs = rte_zmalloc("fm10k",
3144                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3145         if (dev->data->mac_addrs == NULL) {
3146                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3147                 return -ENOMEM;
3148         }
3149
3150         diag = fm10k_read_mac_addr(hw);
3151
3152         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3153                         &dev->data->mac_addrs[0]);
3154
3155         if (diag != FM10K_SUCCESS ||
3156                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3157
3158                 /* Generate a random addr */
3159                 rte_eth_random_addr(hw->mac.addr);
3160                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3161                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3162                 &dev->data->mac_addrs[0]);
3163         }
3164
3165         /* Pass the information to the rte_eth_dev_close() that it should also
3166          * release the private port resources.
3167          */
3168         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
3169
3170         /* Reset the hw statistics */
3171         diag = fm10k_stats_reset(dev);
3172         if (diag != 0) {
3173                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3174                 return diag;
3175         }
3176
3177         /* Reset the hw */
3178         diag = fm10k_reset_hw(hw);
3179         if (diag != FM10K_SUCCESS) {
3180                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3181                 return -EIO;
3182         }
3183
3184         /* Setup mailbox service */
3185         diag = fm10k_setup_mbx_service(hw);
3186         if (diag != FM10K_SUCCESS) {
3187                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3188                 return -EIO;
3189         }
3190
3191         /*PF/VF has different interrupt handling mechanism */
3192         if (hw->mac.type == fm10k_mac_pf) {
3193                 /* register callback func to eal lib */
3194                 rte_intr_callback_register(intr_handle,
3195                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3196
3197                 /* enable MISC interrupt */
3198                 fm10k_dev_enable_intr_pf(dev);
3199         } else { /* VF */
3200                 rte_intr_callback_register(intr_handle,
3201                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3202
3203                 fm10k_dev_enable_intr_vf(dev);
3204         }
3205
3206         /* Enable intr after callback registered */
3207         rte_intr_enable(intr_handle);
3208
3209         hw->mac.ops.update_int_moderator(hw);
3210
3211         /* Make sure Switch Manager is ready before going forward. */
3212         if (hw->mac.type == fm10k_mac_pf) {
3213                 bool switch_ready = false;
3214
3215                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3216                         fm10k_mbx_lock(hw);
3217                         hw->mac.ops.get_host_state(hw, &switch_ready);
3218                         fm10k_mbx_unlock(hw);
3219                         if (switch_ready == true)
3220                                 break;
3221                         /* Delay some time to acquire async LPORT_MAP info. */
3222                         rte_delay_us(WAIT_SWITCH_MSG_US);
3223                 }
3224
3225                 if (switch_ready == false) {
3226                         PMD_INIT_LOG(ERR, "switch is not ready");
3227                         return -1;
3228                 }
3229         }
3230
3231         /*
3232          * Below function will trigger operations on mailbox, acquire lock to
3233          * avoid race condition from interrupt handler. Operations on mailbox
3234          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3235          * will handle and generate an interrupt to our side. Then,  FIFO in
3236          * mailbox will be touched.
3237          */
3238         fm10k_mbx_lock(hw);
3239         /* Enable port first */
3240         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3241                                         MAX_LPORT_NUM, 1);
3242
3243         /* Set unicast mode by default. App can change to other mode in other
3244          * API func.
3245          */
3246         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3247                                         FM10K_XCAST_MODE_NONE);
3248
3249         fm10k_mbx_unlock(hw);
3250
3251         /* Make sure default VID is ready before going forward. */
3252         if (hw->mac.type == fm10k_mac_pf) {
3253                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3254                         if (hw->mac.default_vid)
3255                                 break;
3256                         /* Delay some time to acquire async port VLAN info. */
3257                         rte_delay_us(WAIT_SWITCH_MSG_US);
3258                 }
3259
3260                 if (!hw->mac.default_vid) {
3261                         PMD_INIT_LOG(ERR, "default VID is not ready");
3262                         return -1;
3263                 }
3264         }
3265
3266         /* Add default mac address */
3267         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3268                 MAIN_VSI_POOL_NUMBER);
3269
3270         return 0;
3271 }
3272
3273 static int
3274 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3275 {
3276         PMD_INIT_FUNC_TRACE();
3277
3278         /* only uninitialize in the primary process */
3279         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3280                 return 0;
3281
3282         /* safe to close dev here */
3283         fm10k_dev_close(dev);
3284
3285         return 0;
3286 }
3287
3288 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3289         struct rte_pci_device *pci_dev)
3290 {
3291         return rte_eth_dev_pci_generic_probe(pci_dev,
3292                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3293 }
3294
3295 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3296 {
3297         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3298 }
3299
3300 /*
3301  * The set of PCI devices this driver supports. This driver will enable both PF
3302  * and SRIOV-VF devices.
3303  */
3304 static const struct rte_pci_id pci_id_fm10k_map[] = {
3305         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3306         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3307         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3308         { .vendor_id = 0, /* sentinel */ },
3309 };
3310
3311 static struct rte_pci_driver rte_pmd_fm10k = {
3312         .id_table = pci_id_fm10k_map,
3313         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3314         .probe = eth_fm10k_pci_probe,
3315         .remove = eth_fm10k_pci_remove,
3316 };
3317
3318 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3319 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3320 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3321
3322 RTE_INIT(fm10k_init_log)
3323 {
3324         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3325         if (fm10k_logtype_init >= 0)
3326                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3327         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3328         if (fm10k_logtype_driver >= 0)
3329                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3330
3331 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
3332         fm10k_logtype_rx = rte_log_register("pmd.net.fm10k.rx");
3333         if (fm10k_logtype_rx >= 0)
3334                 rte_log_set_level(fm10k_logtype_rx, RTE_LOG_DEBUG);
3335 #endif
3336
3337 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX
3338         fm10k_logtype_tx = rte_log_register("pmd.net.fm10k.tx");
3339         if (fm10k_logtype_tx >= 0)
3340                 rte_log_set_level(fm10k_logtype_tx, RTE_LOG_DEBUG);
3341 #endif
3342
3343 #ifdef RTE_LIBRTE_FM10K_DEBUG_TX_FREE
3344         fm10k_logtype_tx_free = rte_log_register("pmd.net.fm10k.tx_free");
3345         if (fm10k_logtype_tx_free >= 0)
3346                 rte_log_set_level(fm10k_logtype_tx_free, RTE_LOG_DEBUG);
3347 #endif
3348 }