drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 /* fls = find last set bit = 32 minus the number of leading zeros */
 468 #ifndef fls
 469 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 470 #endif
 471
 472 static void
 473 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 474 {
 475         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 476         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 477         uint32_t i;
 478
 479         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 480
 481         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 482                 if (!vmdq_conf->pool_map[i].pools)
 483                         continue;
 484                 fm10k_mbx_lock(hw);
 485                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 486                 fm10k_mbx_unlock(hw);
 487         }
 488 }
 489
 490 static void
 491 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 492 {
 493         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 494
 495         /* Add default mac address */
 496         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 497                 MAIN_VSI_POOL_NUMBER);
 498 }
 499
 500 static void
 501 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 502 {
 503         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 504         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 505         uint32_t mrqc, *key, i, reta, j;
 506         uint64_t hf;
 507
 508 #define RSS_KEY_SIZE 40
 509         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 510                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 511                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 512                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 513                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 514                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 515         };
 516
 517         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 518                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 519                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 520                 return;
 521         }
 522
 523         /* random key is rss_intel_key (default) or user provided (rss_key) */
 524         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 525                 key = (uint32_t *)rss_intel_key;
 526         else
 527                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 528
 529         /* Now fill our hash function seeds, 4 bytes at a time */
 530         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 531                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 532
 533         /*
 534          * Fill in redirection table
 535          * The byte-swap is needed because NIC registers are in
 536          * little-endian order.
 537          */
 538         reta = 0;
 539         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 540                 if (j == dev->data->nb_rx_queues)
 541                         j = 0;
 542                 reta = (reta << CHAR_BIT) | j;
 543                 if ((i & 3) == 3)
 544                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 545                                         rte_bswap32(reta));
 546         }
 547
 548         /*
 549          * Generate RSS hash based on packet types, TCP/UDP
 550          * port numbers and/or IPv4/v6 src and dst addresses
 551          */
 552         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 553         mrqc = 0;
 554         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 555         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 556         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 557         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 558         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 559         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 560         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 561         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 562         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 563
 564         if (mrqc == 0) {
 565                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 566                         "supported", hf);
 567                 return;
 568         }
 569
 570         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 571 }
 572
 573 static void
 574 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 575 {
 576         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 577         uint32_t i;
 578
 579         for (i = 0; i < nb_lport_new; i++) {
 580                 /* Set unicast mode by default. App can change
 581                  * to other mode in other API func.
 582                  */
 583                 fm10k_mbx_lock(hw);
 584                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 585                         FM10K_XCAST_MODE_NONE);
 586                 fm10k_mbx_unlock(hw);
 587         }
 588 }
 589
 590 static void
 591 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 592 {
 593         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 594         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 595         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 596         struct fm10k_macvlan_filter_info *macvlan;
 597         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 598         uint16_t nb_lport_new;
 599
 600         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 601         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 602
 603         fm10k_dev_rss_configure(dev);
 604
 605         /* only PF supports VMDQ */
 606         if (hw->mac.type != fm10k_mac_pf)
 607                 return;
 608
 609         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 610                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 611
 612         /* no pool number change, no need to update logic port and VLAN/MAC */
 613         if (macvlan->nb_queue_pools == nb_queue_pools)
 614                 return;
 615
 616         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 617         fm10k_dev_logic_port_update(dev, nb_lport_new);
 618
 619         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 620         memset(dev->data->mac_addrs, 0,
 621                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 622         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 623                 &dev->data->mac_addrs[0]);
 624         memset(macvlan, 0, sizeof(*macvlan));
 625         macvlan->nb_queue_pools = nb_queue_pools;
 626
 627         if (nb_queue_pools)
 628                 fm10k_dev_vmdq_rx_configure(dev);
 629         else
 630                 fm10k_dev_pf_main_vsi_reset(dev);
 631 }
 632
 633 static int
 634 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 635 {
 636         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 637         int i, ret;
 638         struct fm10k_tx_queue *txq;
 639         uint64_t base_addr;
 640         uint32_t size;
 641
 642         /* Disable TXINT to avoid possible interrupt */
 643         for (i = 0; i < hw->mac.max_queues; i++)
 644                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 645                                 3 << FM10K_TXINT_TIMER_SHIFT);
 646
 647         /* Setup TX queue */
 648         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 649                 txq = dev->data->tx_queues[i];
 650                 base_addr = txq->hw_ring_phys_addr;
 651                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 652
 653                 /* disable queue to avoid issues while updating state */
 654                 ret = tx_queue_disable(hw, i);
 655                 if (ret) {
 656                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 657                         return -1;
 658                 }
 659                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 660                  * register is read-only for VF.
 661                  */
 662                 if (fm10k_check_ftag(dev->device->devargs)) {
 663                         if (hw->mac.type == fm10k_mac_pf) {
 664                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 665                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 666                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 667                         } else {
 668                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 669                                 return -ENOTSUP;
 670                         }
 671                 }
 672
 673                 /* set location and size for descriptor ring */
 674                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 675                                 base_addr & UINT64_LOWER_32BITS_MASK);
 676                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 677                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 678                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 679
 680                 /* assign default SGLORT for each TX queue by PF */
 681                 if (hw->mac.type == fm10k_mac_pf)
 682                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 683         }
 684
 685         /* set up vector or scalar TX function as appropriate */
 686         fm10k_set_tx_function(dev);
 687
 688         return 0;
 689 }
 690
 691 static int
 692 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 693 {
 694         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 695         struct fm10k_macvlan_filter_info *macvlan;
 696         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 697         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 698         int i, ret;
 699         struct fm10k_rx_queue *rxq;
 700         uint64_t base_addr;
 701         uint32_t size;
 702         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 703         uint32_t logic_port = hw->mac.dglort_map;
 704         uint16_t buf_size;
 705         uint16_t queue_stride = 0;
 706
 707         /* enable RXINT for interrupt mode */
 708         i = 0;
 709         if (rte_intr_dp_is_en(intr_handle)) {
 710                 for (; i < dev->data->nb_rx_queues; i++) {
 711                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 712                         if (hw->mac.type == fm10k_mac_pf)
 713                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 714                                         FM10K_ITR_AUTOMASK |
 715                                         FM10K_ITR_MASK_CLEAR);
 716                         else
 717                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 718                                         FM10K_ITR_AUTOMASK |
 719                                         FM10K_ITR_MASK_CLEAR);
 720                 }
 721         }
 722         /* Disable other RXINT to avoid possible interrupt */
 723         for (; i < hw->mac.max_queues; i++)
 724                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 725                         3 << FM10K_RXINT_TIMER_SHIFT);
 726
 727         /* Setup RX queues */
 728         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 729                 rxq = dev->data->rx_queues[i];
 730                 base_addr = rxq->hw_ring_phys_addr;
 731                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 732
 733                 /* disable queue to avoid issues while updating state */
 734                 ret = rx_queue_disable(hw, i);
 735                 if (ret) {
 736                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 737                         return -1;
 738                 }
 739
 740                 /* Setup the Base and Length of the Rx Descriptor Ring */
 741                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 742                                 base_addr & UINT64_LOWER_32BITS_MASK);
 743                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 744                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 745                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 746
 747                 /* Configure the Rx buffer size for one buff without split */
 748                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 749                         RTE_PKTMBUF_HEADROOM);
 750                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 751                  * reserved for this purpose, and the worst case could be 511B.
 752                  * But SRR reg assumes all buffers have the same size. In order
 753                  * to fill the gap, we'll have to consider the worst case and
 754                  * assume 512B is reserved. If we don't do so, it's possible
 755                  * for HW to overwrite data to next mbuf.
 756                  */
 757                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 758
 759                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 760                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 761                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 762
 763                 /* It adds dual VLAN length for supporting dual VLAN */
 764                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 765                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 766                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 767                         uint32_t reg;
 768                         dev->data->scattered_rx = 1;
 769                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 770                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 771                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 772                 }
 773
 774                 /* Enable drop on empty, it's RO for VF */
 775                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 776                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 777
 778                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 779                 FM10K_WRITE_FLUSH(hw);
 780         }
 781
 782         /* Configure VMDQ/RSS if applicable */
 783         fm10k_dev_mq_rx_configure(dev);
 784
 785         /* Decide the best RX function */
 786         fm10k_set_rx_function(dev);
 787
 788         /* update RX_SGLORT for loopback suppress*/
 789         if (hw->mac.type != fm10k_mac_pf)
 790                 return 0;
 791         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 792         if (macvlan->nb_queue_pools)
 793                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 794         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 795                 if (i && queue_stride && !(i % queue_stride))
 796                         logic_port++;
 797                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 798         }
 799
 800         return 0;
 801 }
 802
 803 static int
 804 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 805 {
 806         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 807         int err;
 808         uint32_t reg;
 809         struct fm10k_rx_queue *rxq;
 810
 811         PMD_INIT_FUNC_TRACE();
 812
 813         rxq = dev->data->rx_queues[rx_queue_id];
 814         err = rx_queue_reset(rxq);
 815         if (err == -ENOMEM) {
 816                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 817                 return err;
 818         } else if (err == -EINVAL) {
 819                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 820                         " %d", err);
 821                 return err;
 822         }
 823
 824         /* Setup the HW Rx Head and Tail Descriptor Pointers
 825          * Note: this must be done AFTER the queue is enabled on real
 826          * hardware, but BEFORE the queue is enabled when using the
 827          * emulation platform. Do it in both places for now and remove
 828          * this comment and the following two register writes when the
 829          * emulation platform is no longer being used.
 830          */
 831         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 832         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 833
 834         /* Set PF ownership flag for PF devices */
 835         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 836         if (hw->mac.type == fm10k_mac_pf)
 837                 reg |= FM10K_RXQCTL_PF;
 838         reg |= FM10K_RXQCTL_ENABLE;
 839         /* enable RX queue */
 840         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 841         FM10K_WRITE_FLUSH(hw);
 842
 843         /* Setup the HW Rx Head and Tail Descriptor Pointers
 844          * Note: this must be done AFTER the queue is enabled
 845          */
 846         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 847         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 848         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 849
 850         return 0;
 851 }
 852
 853 static int
 854 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 855 {
 856         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 857
 858         PMD_INIT_FUNC_TRACE();
 859
 860         /* Disable RX queue */
 861         rx_queue_disable(hw, rx_queue_id);
 862
 863         /* Free mbuf and clean HW ring */
 864         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 865         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 866
 867         return 0;
 868 }
 869
 870 static int
 871 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 872 {
 873         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 874         /** @todo - this should be defined in the shared code */
 875 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 876         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 877         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 878
 879         PMD_INIT_FUNC_TRACE();
 880
 881         q->ops->reset(q);
 882
 883         /* reset head and tail pointers */
 884         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 885         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 886
 887         /* enable TX queue */
 888         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 889                                 FM10K_TXDCTL_ENABLE | txdctl);
 890         FM10K_WRITE_FLUSH(hw);
 891         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 892
 893         return 0;
 894 }
 895
 896 static int
 897 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 898 {
 899         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 900
 901         PMD_INIT_FUNC_TRACE();
 902
 903         tx_queue_disable(hw, tx_queue_id);
 904         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 905         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 906
 907         return 0;
 908 }
 909
 910 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 911 {
 912         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 913                 != FM10K_DGLORTMAP_NONE);
 914 }
 915
 916 static void
 917 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 918 {
 919         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 920         int status;
 921
 922         PMD_INIT_FUNC_TRACE();
 923
 924         /* Return if it didn't acquire valid glort range */
 925         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 926                 return;
 927
 928         fm10k_mbx_lock(hw);
 929         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 930                                 FM10K_XCAST_MODE_PROMISC);
 931         fm10k_mbx_unlock(hw);
 932
 933         if (status != FM10K_SUCCESS)
 934                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 935 }
 936
 937 static void
 938 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 939 {
 940         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 941         uint8_t mode;
 942         int status;
 943
 944         PMD_INIT_FUNC_TRACE();
 945
 946         /* Return if it didn't acquire valid glort range */
 947         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 948                 return;
 949
 950         if (dev->data->all_multicast == 1)
 951                 mode = FM10K_XCAST_MODE_ALLMULTI;
 952         else
 953                 mode = FM10K_XCAST_MODE_NONE;
 954
 955         fm10k_mbx_lock(hw);
 956         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 957                                 mode);
 958         fm10k_mbx_unlock(hw);
 959
 960         if (status != FM10K_SUCCESS)
 961                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 962 }
 963
 964 static void
 965 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 966 {
 967         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 968         int status;
 969
 970         PMD_INIT_FUNC_TRACE();
 971
 972         /* Return if it didn't acquire valid glort range */
 973         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 974                 return;
 975
 976         /* If promiscuous mode is enabled, it doesn't make sense to enable
 977          * allmulticast and disable promiscuous since fm10k only can select
 978          * one of the modes.
 979          */
 980         if (dev->data->promiscuous) {
 981                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 982                         "needn't enable allmulticast");
 983                 return;
 984         }
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 FM10K_XCAST_MODE_ALLMULTI);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         if (dev->data->promiscuous) {
1008                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1009                         "since promisc mode is enabled");
1010                 return;
1011         }
1012
1013         fm10k_mbx_lock(hw);
1014         /* Change mode to unicast mode */
1015         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1016                                 FM10K_XCAST_MODE_NONE);
1017         fm10k_mbx_unlock(hw);
1018
1019         if (status != FM10K_SUCCESS)
1020                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1021 }
1022
1023 static void
1024 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1025 {
1026         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1027         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1028         uint16_t nb_queue_pools;
1029         struct fm10k_macvlan_filter_info *macvlan;
1030
1031         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1032         nb_queue_pools = macvlan->nb_queue_pools;
1033         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1034         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1035
1036         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1037         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1038         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1039                         hw->mac.dglort_map;
1040         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1041         /* Configure VMDQ/RSS DGlort Decoder */
1042         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1043
1044         /* Flow Director configurations, only queue number is valid. */
1045         dglortdec = fls(dev->data->nb_rx_queues - 1);
1046         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1047                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1048         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1049         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1050
1051         /* Invalidate all other GLORT entries */
1052         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1053                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1054                                 FM10K_DGLORTMAP_NONE);
1055 }
1056
1057 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1058 static int
1059 fm10k_dev_start(struct rte_eth_dev *dev)
1060 {
1061         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1062         int i, diag;
1063
1064         PMD_INIT_FUNC_TRACE();
1065
1066         /* stop, init, then start the hw */
1067         diag = fm10k_stop_hw(hw);
1068         if (diag != FM10K_SUCCESS) {
1069                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1070                 return -EIO;
1071         }
1072
1073         diag = fm10k_init_hw(hw);
1074         if (diag != FM10K_SUCCESS) {
1075                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1076                 return -EIO;
1077         }
1078
1079         diag = fm10k_start_hw(hw);
1080         if (diag != FM10K_SUCCESS) {
1081                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1082                 return -EIO;
1083         }
1084
1085         diag = fm10k_dev_tx_init(dev);
1086         if (diag) {
1087                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1088                 return diag;
1089         }
1090
1091         if (fm10k_dev_rxq_interrupt_setup(dev))
1092                 return -EIO;
1093
1094         diag = fm10k_dev_rx_init(dev);
1095         if (diag) {
1096                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1097                 return diag;
1098         }
1099
1100         if (hw->mac.type == fm10k_mac_pf)
1101                 fm10k_dev_dglort_map_configure(dev);
1102
1103         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1104                 struct fm10k_rx_queue *rxq;
1105                 rxq = dev->data->rx_queues[i];
1106
1107                 if (rxq->rx_deferred_start)
1108                         continue;
1109                 diag = fm10k_dev_rx_queue_start(dev, i);
1110                 if (diag != 0) {
1111                         int j;
1112                         for (j = 0; j < i; ++j)
1113                                 rx_queue_clean(dev->data->rx_queues[j]);
1114                         return diag;
1115                 }
1116         }
1117
1118         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1119                 struct fm10k_tx_queue *txq;
1120                 txq = dev->data->tx_queues[i];
1121
1122                 if (txq->tx_deferred_start)
1123                         continue;
1124                 diag = fm10k_dev_tx_queue_start(dev, i);
1125                 if (diag != 0) {
1126                         int j;
1127                         for (j = 0; j < i; ++j)
1128                                 tx_queue_clean(dev->data->tx_queues[j]);
1129                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1130                                 rx_queue_clean(dev->data->rx_queues[j]);
1131                         return diag;
1132                 }
1133         }
1134
1135         /* Update default vlan when not in VMDQ mode */
1136         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1137                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1138
1139         fm10k_link_update(dev, 0);
1140
1141         return 0;
1142 }
1143
1144 static void
1145 fm10k_dev_stop(struct rte_eth_dev *dev)
1146 {
1147         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1148         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1149         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1150         int i;
1151
1152         PMD_INIT_FUNC_TRACE();
1153
1154         if (dev->data->tx_queues)
1155                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1156                         fm10k_dev_tx_queue_stop(dev, i);
1157
1158         if (dev->data->rx_queues)
1159                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1160                         fm10k_dev_rx_queue_stop(dev, i);
1161
1162         /* Disable datapath event */
1163         if (rte_intr_dp_is_en(intr_handle)) {
1164                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1165                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1166                                 3 << FM10K_RXINT_TIMER_SHIFT);
1167                         if (hw->mac.type == fm10k_mac_pf)
1168                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1169                                         FM10K_ITR_MASK_SET);
1170                         else
1171                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1172                                         FM10K_ITR_MASK_SET);
1173                 }
1174         }
1175         /* Clean datapath event and queue/vec mapping */
1176         rte_intr_efd_disable(intr_handle);
1177         rte_free(intr_handle->intr_vec);
1178         intr_handle->intr_vec = NULL;
1179 }
1180
1181 static void
1182 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1183 {
1184         int i;
1185
1186         PMD_INIT_FUNC_TRACE();
1187
1188         if (dev->data->tx_queues) {
1189                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1190                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1191
1192                         tx_queue_free(txq);
1193                 }
1194         }
1195
1196         if (dev->data->rx_queues) {
1197                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1198                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1199         }
1200 }
1201
1202 static void
1203 fm10k_dev_close(struct rte_eth_dev *dev)
1204 {
1205         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1206
1207         PMD_INIT_FUNC_TRACE();
1208
1209         fm10k_mbx_lock(hw);
1210         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1211                 MAX_LPORT_NUM, false);
1212         fm10k_mbx_unlock(hw);
1213
1214         /* allow 100ms for device to quiesce */
1215         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1216
1217         /* Stop mailbox service first */
1218         fm10k_close_mbx_service(hw);
1219         fm10k_dev_stop(dev);
1220         fm10k_dev_queue_release(dev);
1221         fm10k_stop_hw(hw);
1222 }
1223
1224 static int
1225 fm10k_link_update(struct rte_eth_dev *dev,
1226         __rte_unused int wait_to_complete)
1227 {
1228         struct fm10k_dev_info *dev_info =
1229                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1230         PMD_INIT_FUNC_TRACE();
1231
1232         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1233         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1234         dev->data->dev_link.link_status =
1235                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1236         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1237
1238         return 0;
1239 }
1240
1241 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1242         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1243 {
1244         unsigned i, q;
1245         unsigned count = 0;
1246
1247         if (xstats_names != NULL) {
1248                 /* Note: limit checked in rte_eth_xstats_names() */
1249
1250                 /* Global stats */
1251                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1252                         snprintf(xstats_names[count].name,
1253                                 sizeof(xstats_names[count].name),
1254                                 "%s", fm10k_hw_stats_strings[count].name);
1255                         count++;
1256                 }
1257
1258                 /* PF queue stats */
1259                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1260                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1261                                 snprintf(xstats_names[count].name,
1262                                         sizeof(xstats_names[count].name),
1263                                         "rx_q%u_%s", q,
1264                                         fm10k_hw_stats_rx_q_strings[i].name);
1265                                 count++;
1266                         }
1267                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1268                                 snprintf(xstats_names[count].name,
1269                                         sizeof(xstats_names[count].name),
1270                                         "tx_q%u_%s", q,
1271                                         fm10k_hw_stats_tx_q_strings[i].name);
1272                                 count++;
1273                         }
1274                 }
1275         }
1276         return FM10K_NB_XSTATS;
1277 }
1278
1279 static int
1280 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1281                  unsigned n)
1282 {
1283         struct fm10k_hw_stats *hw_stats =
1284                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1285         unsigned i, q, count = 0;
1286
1287         if (n < FM10K_NB_XSTATS)
1288                 return FM10K_NB_XSTATS;
1289
1290         /* Global stats */
1291         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1292                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1293                         fm10k_hw_stats_strings[count].offset);
1294                 xstats[count].id = count;
1295                 count++;
1296         }
1297
1298         /* PF queue stats */
1299         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1300                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1301                         xstats[count].value =
1302                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1303                                 fm10k_hw_stats_rx_q_strings[i].offset);
1304                         xstats[count].id = count;
1305                         count++;
1306                 }
1307                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1308                         xstats[count].value =
1309                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1310                                 fm10k_hw_stats_tx_q_strings[i].offset);
1311                         xstats[count].id = count;
1312                         count++;
1313                 }
1314         }
1315
1316         return FM10K_NB_XSTATS;
1317 }
1318
1319 static int
1320 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1321 {
1322         uint64_t ipackets, opackets, ibytes, obytes;
1323         struct fm10k_hw *hw =
1324                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1325         struct fm10k_hw_stats *hw_stats =
1326                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1327         int i;
1328
1329         PMD_INIT_FUNC_TRACE();
1330
1331         fm10k_update_hw_stats(hw, hw_stats);
1332
1333         ipackets = opackets = ibytes = obytes = 0;
1334         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1335                 (i < hw->mac.max_queues); ++i) {
1336                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1337                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1338                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1339                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1340                 ipackets += stats->q_ipackets[i];
1341                 opackets += stats->q_opackets[i];
1342                 ibytes   += stats->q_ibytes[i];
1343                 obytes   += stats->q_obytes[i];
1344         }
1345         stats->ipackets = ipackets;
1346         stats->opackets = opackets;
1347         stats->ibytes = ibytes;
1348         stats->obytes = obytes;
1349         return 0;
1350 }
1351
1352 static void
1353 fm10k_stats_reset(struct rte_eth_dev *dev)
1354 {
1355         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1356         struct fm10k_hw_stats *hw_stats =
1357                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1358
1359         PMD_INIT_FUNC_TRACE();
1360
1361         memset(hw_stats, 0, sizeof(*hw_stats));
1362         fm10k_rebind_hw_stats(hw, hw_stats);
1363 }
1364
1365 static void
1366 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1367         struct rte_eth_dev_info *dev_info)
1368 {
1369         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1370         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1371
1372         PMD_INIT_FUNC_TRACE();
1373
1374         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1375         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1376         dev_info->max_rx_queues      = hw->mac.max_queues;
1377         dev_info->max_tx_queues      = hw->mac.max_queues;
1378         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1379         dev_info->max_hash_mac_addrs = 0;
1380         dev_info->max_vfs            = pdev->max_vfs;
1381         dev_info->vmdq_pool_base     = 0;
1382         dev_info->vmdq_queue_base    = 0;
1383         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1384         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1385         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1386         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1387                                     dev_info->rx_queue_offload_capa;
1388         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1389         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1390                                     dev_info->tx_queue_offload_capa;
1391
1392         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1393         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1394
1395         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1396                 .rx_thresh = {
1397                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1398                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1399                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1400                 },
1401                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1402                 .rx_drop_en = 0,
1403                 .offloads = 0,
1404         };
1405
1406         dev_info->default_txconf = (struct rte_eth_txconf) {
1407                 .tx_thresh = {
1408                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1409                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1410                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1411                 },
1412                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1413                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1414                 .offloads = 0,
1415         };
1416
1417         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1418                 .nb_max = FM10K_MAX_RX_DESC,
1419                 .nb_min = FM10K_MIN_RX_DESC,
1420                 .nb_align = FM10K_MULT_RX_DESC,
1421         };
1422
1423         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1424                 .nb_max = FM10K_MAX_TX_DESC,
1425                 .nb_min = FM10K_MIN_TX_DESC,
1426                 .nb_align = FM10K_MULT_TX_DESC,
1427                 .nb_seg_max = FM10K_TX_MAX_SEG,
1428                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1429         };
1430
1431         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1432                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1433                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1434 }
1435
1436 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1437 static const uint32_t *
1438 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1439 {
1440         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1441             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1442                 static uint32_t ptypes[] = {
1443                         /* refers to rx_desc_to_ol_flags() */
1444                         RTE_PTYPE_L2_ETHER,
1445                         RTE_PTYPE_L3_IPV4,
1446                         RTE_PTYPE_L3_IPV4_EXT,
1447                         RTE_PTYPE_L3_IPV6,
1448                         RTE_PTYPE_L3_IPV6_EXT,
1449                         RTE_PTYPE_L4_TCP,
1450                         RTE_PTYPE_L4_UDP,
1451                         RTE_PTYPE_UNKNOWN
1452                 };
1453
1454                 return ptypes;
1455         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1456                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1457                 static uint32_t ptypes_vec[] = {
1458                         /* refers to fm10k_desc_to_pktype_v() */
1459                         RTE_PTYPE_L3_IPV4,
1460                         RTE_PTYPE_L3_IPV4_EXT,
1461                         RTE_PTYPE_L3_IPV6,
1462                         RTE_PTYPE_L3_IPV6_EXT,
1463                         RTE_PTYPE_L4_TCP,
1464                         RTE_PTYPE_L4_UDP,
1465                         RTE_PTYPE_TUNNEL_GENEVE,
1466                         RTE_PTYPE_TUNNEL_NVGRE,
1467                         RTE_PTYPE_TUNNEL_VXLAN,
1468                         RTE_PTYPE_TUNNEL_GRE,
1469                         RTE_PTYPE_UNKNOWN
1470                 };
1471
1472                 return ptypes_vec;
1473         }
1474
1475         return NULL;
1476 }
1477 #else
1478 static const uint32_t *
1479 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1480 {
1481         return NULL;
1482 }
1483 #endif
1484
1485 static int
1486 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1487 {
1488         s32 result;
1489         uint16_t mac_num = 0;
1490         uint32_t vid_idx, vid_bit, mac_index;
1491         struct fm10k_hw *hw;
1492         struct fm10k_macvlan_filter_info *macvlan;
1493         struct rte_eth_dev_data *data = dev->data;
1494
1495         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1496         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1497
1498         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1499                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1500                 return -EINVAL;
1501         }
1502
1503         if (vlan_id > ETH_VLAN_ID_MAX) {
1504                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1505                 return -EINVAL;
1506         }
1507
1508         vid_idx = FM10K_VFTA_IDX(vlan_id);
1509         vid_bit = FM10K_VFTA_BIT(vlan_id);
1510         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1511         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1512                 return 0;
1513         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1514         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1515                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1516                         "in the VLAN filter table");
1517                 return -EINVAL;
1518         }
1519
1520         fm10k_mbx_lock(hw);
1521         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1522         fm10k_mbx_unlock(hw);
1523         if (result != FM10K_SUCCESS) {
1524                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1525                 return -EIO;
1526         }
1527
1528         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1529                         (result == FM10K_SUCCESS); mac_index++) {
1530                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1531                         continue;
1532                 if (mac_num > macvlan->mac_num - 1) {
1533                         PMD_INIT_LOG(ERR, "MAC address number "
1534                                         "not match");
1535                         break;
1536                 }
1537                 fm10k_mbx_lock(hw);
1538                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1539                         data->mac_addrs[mac_index].addr_bytes,
1540                         vlan_id, on, 0);
1541                 fm10k_mbx_unlock(hw);
1542                 mac_num++;
1543         }
1544         if (result != FM10K_SUCCESS) {
1545                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1546                 return -EIO;
1547         }
1548
1549         if (on) {
1550                 macvlan->vlan_num++;
1551                 macvlan->vfta[vid_idx] |= vid_bit;
1552         } else {
1553                 macvlan->vlan_num--;
1554                 macvlan->vfta[vid_idx] &= ~vid_bit;
1555         }
1556         return 0;
1557 }
1558
1559 static int
1560 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1561 {
1562         if (mask & ETH_VLAN_STRIP_MASK) {
1563                 if (!(dev->data->dev_conf.rxmode.offloads &
1564                         DEV_RX_OFFLOAD_VLAN_STRIP))
1565                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1566                                         "always on in fm10k");
1567         }
1568
1569         if (mask & ETH_VLAN_EXTEND_MASK) {
1570                 if (dev->data->dev_conf.rxmode.offloads &
1571                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1572                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1573                                         "supported in fm10k");
1574         }
1575
1576         if (mask & ETH_VLAN_FILTER_MASK) {
1577                 if (!(dev->data->dev_conf.rxmode.offloads &
1578                         DEV_RX_OFFLOAD_VLAN_FILTER))
1579                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1580         }
1581
1582         return 0;
1583 }
1584
1585 /* Add/Remove a MAC address, and update filters to main VSI */
1586 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1587                 const u8 *mac, bool add, uint32_t pool)
1588 {
1589         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1590         struct fm10k_macvlan_filter_info *macvlan;
1591         uint32_t i, j, k;
1592
1593         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1594
1595         if (pool != MAIN_VSI_POOL_NUMBER) {
1596                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1597                         "mac to pool %u", pool);
1598                 return;
1599         }
1600         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1601                 if (!macvlan->vfta[j])
1602                         continue;
1603                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1604                         if (!(macvlan->vfta[j] & (1 << k)))
1605                                 continue;
1606                         if (i + 1 > macvlan->vlan_num) {
1607                                 PMD_INIT_LOG(ERR, "vlan number not match");
1608                                 return;
1609                         }
1610                         fm10k_mbx_lock(hw);
1611                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1612                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1613                         fm10k_mbx_unlock(hw);
1614                         i++;
1615                 }
1616         }
1617 }
1618
1619 /* Add/Remove a MAC address, and update filters to VMDQ */
1620 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1621                 const u8 *mac, bool add, uint32_t pool)
1622 {
1623         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1624         struct fm10k_macvlan_filter_info *macvlan;
1625         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1626         uint32_t i;
1627
1628         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1629         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1630
1631         if (pool > macvlan->nb_queue_pools) {
1632                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1633                         " Max pool is %u",
1634                         pool, macvlan->nb_queue_pools);
1635                 return;
1636         }
1637         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1638                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1639                         continue;
1640                 fm10k_mbx_lock(hw);
1641                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1642                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1643                 fm10k_mbx_unlock(hw);
1644         }
1645 }
1646
1647 /* Add/Remove a MAC address, and update filters */
1648 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1649                 const u8 *mac, bool add, uint32_t pool)
1650 {
1651         struct fm10k_macvlan_filter_info *macvlan;
1652
1653         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1654
1655         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1656                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1657         else
1658                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1659
1660         if (add)
1661                 macvlan->mac_num++;
1662         else
1663                 macvlan->mac_num--;
1664 }
1665
1666 /* Add a MAC address, and update filters */
1667 static int
1668 fm10k_macaddr_add(struct rte_eth_dev *dev,
1669                 struct ether_addr *mac_addr,
1670                 uint32_t index,
1671                 uint32_t pool)
1672 {
1673         struct fm10k_macvlan_filter_info *macvlan;
1674
1675         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1676         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1677         macvlan->mac_vmdq_id[index] = pool;
1678         return 0;
1679 }
1680
1681 /* Remove a MAC address, and update filters */
1682 static void
1683 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1684 {
1685         struct rte_eth_dev_data *data = dev->data;
1686         struct fm10k_macvlan_filter_info *macvlan;
1687
1688         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1689         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1690                         FALSE, macvlan->mac_vmdq_id[index]);
1691         macvlan->mac_vmdq_id[index] = 0;
1692 }
1693
1694 static inline int
1695 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1696 {
1697         if ((request < min) || (request > max) || ((request % mult) != 0))
1698                 return -1;
1699         else
1700                 return 0;
1701 }
1702
1703
1704 static inline int
1705 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1706 {
1707         if ((request < min) || (request > max) || ((div % request) != 0))
1708                 return -1;
1709         else
1710                 return 0;
1711 }
1712
1713 static inline int
1714 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1715 {
1716         uint16_t rx_free_thresh;
1717
1718         if (conf->rx_free_thresh == 0)
1719                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1720         else
1721                 rx_free_thresh = conf->rx_free_thresh;
1722
1723         /* make sure the requested threshold satisfies the constraints */
1724         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1725                         FM10K_RX_FREE_THRESH_MAX(q),
1726                         FM10K_RX_FREE_THRESH_DIV(q),
1727                         rx_free_thresh)) {
1728                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1729                         "less than or equal to %u, "
1730                         "greater than or equal to %u, "
1731                         "and a divisor of %u",
1732                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1733                         FM10K_RX_FREE_THRESH_MIN(q),
1734                         FM10K_RX_FREE_THRESH_DIV(q));
1735                 return -EINVAL;
1736         }
1737
1738         q->alloc_thresh = rx_free_thresh;
1739         q->drop_en = conf->rx_drop_en;
1740         q->rx_deferred_start = conf->rx_deferred_start;
1741
1742         return 0;
1743 }
1744
1745 /*
1746  * Hardware requires specific alignment for Rx packet buffers. At
1747  * least one of the following two conditions must be satisfied.
1748  *  1. Address is 512B aligned
1749  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1750  *
1751  * As such, the driver may need to adjust the DMA address within the
1752  * buffer by up to 512B.
1753  *
1754  * return 1 if the element size is valid, otherwise return 0.
1755  */
1756 static int
1757 mempool_element_size_valid(struct rte_mempool *mp)
1758 {
1759         uint32_t min_size;
1760
1761         /* elt_size includes mbuf header and headroom */
1762         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1763                         RTE_PKTMBUF_HEADROOM;
1764
1765         /* account for up to 512B of alignment */
1766         min_size -= FM10K_RX_DATABUF_ALIGN;
1767
1768         /* sanity check for overflow */
1769         if (min_size > mp->elt_size)
1770                 return 0;
1771
1772         /* size is valid */
1773         return 1;
1774 }
1775
1776 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1777 {
1778         RTE_SET_USED(dev);
1779
1780         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1781 }
1782
1783 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1784 {
1785         RTE_SET_USED(dev);
1786
1787         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1788                            DEV_RX_OFFLOAD_VLAN_FILTER |
1789                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1790                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1791                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1792                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1793                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1794 }
1795
1796 static int
1797 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1798         uint16_t nb_desc, unsigned int socket_id,
1799         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1800 {
1801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1802         struct fm10k_dev_info *dev_info =
1803                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1804         struct fm10k_rx_queue *q;
1805         const struct rte_memzone *mz;
1806         uint64_t offloads;
1807
1808         PMD_INIT_FUNC_TRACE();
1809
1810         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1811
1812         /* make sure the mempool element size can account for alignment. */
1813         if (!mempool_element_size_valid(mp)) {
1814                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1815                 return -EINVAL;
1816         }
1817
1818         /* make sure a valid number of descriptors have been requested */
1819         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1820                                 FM10K_MULT_RX_DESC, nb_desc)) {
1821                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1822                         "less than or equal to %"PRIu32", "
1823                         "greater than or equal to %u, "
1824                         "and a multiple of %u",
1825                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1826                         FM10K_MULT_RX_DESC);
1827                 return -EINVAL;
1828         }
1829
1830         /*
1831          * if this queue existed already, free the associated memory. The
1832          * queue cannot be reused in case we need to allocate memory on
1833          * different socket than was previously used.
1834          */
1835         if (dev->data->rx_queues[queue_id] != NULL) {
1836                 rx_queue_free(dev->data->rx_queues[queue_id]);
1837                 dev->data->rx_queues[queue_id] = NULL;
1838         }
1839
1840         /* allocate memory for the queue structure */
1841         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1842                                 socket_id);
1843         if (q == NULL) {
1844                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1845                 return -ENOMEM;
1846         }
1847
1848         /* setup queue */
1849         q->mp = mp;
1850         q->nb_desc = nb_desc;
1851         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1852         q->port_id = dev->data->port_id;
1853         q->queue_id = queue_id;
1854         q->tail_ptr = (volatile uint32_t *)
1855                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1856         q->offloads = offloads;
1857         if (handle_rxconf(q, conf))
1858                 return -EINVAL;
1859
1860         /* allocate memory for the software ring */
1861         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1862                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1863                         RTE_CACHE_LINE_SIZE, socket_id);
1864         if (q->sw_ring == NULL) {
1865                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1866                 rte_free(q);
1867                 return -ENOMEM;
1868         }
1869
1870         /*
1871          * allocate memory for the hardware descriptor ring. A memzone large
1872          * enough to hold the maximum ring size is requested to allow for
1873          * resizing in later calls to the queue setup function.
1874          */
1875         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1876                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1877                                       socket_id);
1878         if (mz == NULL) {
1879                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1880                 rte_free(q->sw_ring);
1881                 rte_free(q);
1882                 return -ENOMEM;
1883         }
1884         q->hw_ring = mz->addr;
1885         q->hw_ring_phys_addr = mz->iova;
1886
1887         /* Check if number of descs satisfied Vector requirement */
1888         if (!rte_is_power_of_2(nb_desc)) {
1889                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1890                                     "preconditions - canceling the feature for "
1891                                     "the whole port[%d]",
1892                              q->queue_id, q->port_id);
1893                 dev_info->rx_vec_allowed = false;
1894         } else
1895                 fm10k_rxq_vec_setup(q);
1896
1897         dev->data->rx_queues[queue_id] = q;
1898         return 0;
1899 }
1900
1901 static void
1902 fm10k_rx_queue_release(void *queue)
1903 {
1904         PMD_INIT_FUNC_TRACE();
1905
1906         rx_queue_free(queue);
1907 }
1908
1909 static inline int
1910 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1911 {
1912         uint16_t tx_free_thresh;
1913         uint16_t tx_rs_thresh;
1914
1915         /* constraint MACROs require that tx_free_thresh is configured
1916          * before tx_rs_thresh */
1917         if (conf->tx_free_thresh == 0)
1918                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1919         else
1920                 tx_free_thresh = conf->tx_free_thresh;
1921
1922         /* make sure the requested threshold satisfies the constraints */
1923         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1924                         FM10K_TX_FREE_THRESH_MAX(q),
1925                         FM10K_TX_FREE_THRESH_DIV(q),
1926                         tx_free_thresh)) {
1927                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1928                         "less than or equal to %u, "
1929                         "greater than or equal to %u, "
1930                         "and a divisor of %u",
1931                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1932                         FM10K_TX_FREE_THRESH_MIN(q),
1933                         FM10K_TX_FREE_THRESH_DIV(q));
1934                 return -EINVAL;
1935         }
1936
1937         q->free_thresh = tx_free_thresh;
1938
1939         if (conf->tx_rs_thresh == 0)
1940                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1941         else
1942                 tx_rs_thresh = conf->tx_rs_thresh;
1943
1944         q->tx_deferred_start = conf->tx_deferred_start;
1945
1946         /* make sure the requested threshold satisfies the constraints */
1947         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1948                         FM10K_TX_RS_THRESH_MAX(q),
1949                         FM10K_TX_RS_THRESH_DIV(q),
1950                         tx_rs_thresh)) {
1951                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1952                         "less than or equal to %u, "
1953                         "greater than or equal to %u, "
1954                         "and a divisor of %u",
1955                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1956                         FM10K_TX_RS_THRESH_MIN(q),
1957                         FM10K_TX_RS_THRESH_DIV(q));
1958                 return -EINVAL;
1959         }
1960
1961         q->rs_thresh = tx_rs_thresh;
1962
1963         return 0;
1964 }
1965
1966 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1967 {
1968         RTE_SET_USED(dev);
1969
1970         return 0;
1971 }
1972
1973 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1974 {
1975         RTE_SET_USED(dev);
1976
1977         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1978                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1979                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1980                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1981                           DEV_TX_OFFLOAD_TCP_TSO);
1982 }
1983
1984 static int
1985 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1986         uint16_t nb_desc, unsigned int socket_id,
1987         const struct rte_eth_txconf *conf)
1988 {
1989         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1990         struct fm10k_tx_queue *q;
1991         const struct rte_memzone *mz;
1992         uint64_t offloads;
1993
1994         PMD_INIT_FUNC_TRACE();
1995
1996         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
1997
1998         /* make sure a valid number of descriptors have been requested */
1999         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2000                                 FM10K_MULT_TX_DESC, nb_desc)) {
2001                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2002                         "less than or equal to %"PRIu32", "
2003                         "greater than or equal to %u, "
2004                         "and a multiple of %u",
2005                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2006                         FM10K_MULT_TX_DESC);
2007                 return -EINVAL;
2008         }
2009
2010         /*
2011          * if this queue existed already, free the associated memory. The
2012          * queue cannot be reused in case we need to allocate memory on
2013          * different socket than was previously used.
2014          */
2015         if (dev->data->tx_queues[queue_id] != NULL) {
2016                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2017
2018                 tx_queue_free(txq);
2019                 dev->data->tx_queues[queue_id] = NULL;
2020         }
2021
2022         /* allocate memory for the queue structure */
2023         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2024                                 socket_id);
2025         if (q == NULL) {
2026                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2027                 return -ENOMEM;
2028         }
2029
2030         /* setup queue */
2031         q->nb_desc = nb_desc;
2032         q->port_id = dev->data->port_id;
2033         q->queue_id = queue_id;
2034         q->offloads = offloads;
2035         q->ops = &def_txq_ops;
2036         q->tail_ptr = (volatile uint32_t *)
2037                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2038         if (handle_txconf(q, conf))
2039                 return -EINVAL;
2040
2041         /* allocate memory for the software ring */
2042         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2043                                         nb_desc * sizeof(struct rte_mbuf *),
2044                                         RTE_CACHE_LINE_SIZE, socket_id);
2045         if (q->sw_ring == NULL) {
2046                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2047                 rte_free(q);
2048                 return -ENOMEM;
2049         }
2050
2051         /*
2052          * allocate memory for the hardware descriptor ring. A memzone large
2053          * enough to hold the maximum ring size is requested to allow for
2054          * resizing in later calls to the queue setup function.
2055          */
2056         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2057                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2058                                       socket_id);
2059         if (mz == NULL) {
2060                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2061                 rte_free(q->sw_ring);
2062                 rte_free(q);
2063                 return -ENOMEM;
2064         }
2065         q->hw_ring = mz->addr;
2066         q->hw_ring_phys_addr = mz->iova;
2067
2068         /*
2069          * allocate memory for the RS bit tracker. Enough slots to hold the
2070          * descriptor index for each RS bit needing to be set are required.
2071          */
2072         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2073                                 ((nb_desc + 1) / q->rs_thresh) *
2074                                 sizeof(uint16_t),
2075                                 RTE_CACHE_LINE_SIZE, socket_id);
2076         if (q->rs_tracker.list == NULL) {
2077                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2078                 rte_free(q->sw_ring);
2079                 rte_free(q);
2080                 return -ENOMEM;
2081         }
2082
2083         dev->data->tx_queues[queue_id] = q;
2084         return 0;
2085 }
2086
2087 static void
2088 fm10k_tx_queue_release(void *queue)
2089 {
2090         struct fm10k_tx_queue *q = queue;
2091         PMD_INIT_FUNC_TRACE();
2092
2093         tx_queue_free(q);
2094 }
2095
2096 static int
2097 fm10k_reta_update(struct rte_eth_dev *dev,
2098                         struct rte_eth_rss_reta_entry64 *reta_conf,
2099                         uint16_t reta_size)
2100 {
2101         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2102         uint16_t i, j, idx, shift;
2103         uint8_t mask;
2104         uint32_t reta;
2105
2106         PMD_INIT_FUNC_TRACE();
2107
2108         if (reta_size > FM10K_MAX_RSS_INDICES) {
2109                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2110                         "(%d) doesn't match the number hardware can supported "
2111                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2112                 return -EINVAL;
2113         }
2114
2115         /*
2116          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2117          * 128-entries in 32 registers
2118          */
2119         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2120                 idx = i / RTE_RETA_GROUP_SIZE;
2121                 shift = i % RTE_RETA_GROUP_SIZE;
2122                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2123                                 BIT_MASK_PER_UINT32);
2124                 if (mask == 0)
2125                         continue;
2126
2127                 reta = 0;
2128                 if (mask != BIT_MASK_PER_UINT32)
2129                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2130
2131                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2132                         if (mask & (0x1 << j)) {
2133                                 if (mask != 0xF)
2134                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2135                                 reta |= reta_conf[idx].reta[shift + j] <<
2136                                                 (CHAR_BIT * j);
2137                         }
2138                 }
2139                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2140         }
2141
2142         return 0;
2143 }
2144
2145 static int
2146 fm10k_reta_query(struct rte_eth_dev *dev,
2147                         struct rte_eth_rss_reta_entry64 *reta_conf,
2148                         uint16_t reta_size)
2149 {
2150         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2151         uint16_t i, j, idx, shift;
2152         uint8_t mask;
2153         uint32_t reta;
2154
2155         PMD_INIT_FUNC_TRACE();
2156
2157         if (reta_size < FM10K_MAX_RSS_INDICES) {
2158                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2159                         "(%d) doesn't match the number hardware can supported "
2160                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2161                 return -EINVAL;
2162         }
2163
2164         /*
2165          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2166          * 128-entries in 32 registers
2167          */
2168         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2169                 idx = i / RTE_RETA_GROUP_SIZE;
2170                 shift = i % RTE_RETA_GROUP_SIZE;
2171                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2172                                 BIT_MASK_PER_UINT32);
2173                 if (mask == 0)
2174                         continue;
2175
2176                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2177                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2178                         if (mask & (0x1 << j))
2179                                 reta_conf[idx].reta[shift + j] = ((reta >>
2180                                         CHAR_BIT * j) & UINT8_MAX);
2181                 }
2182         }
2183
2184         return 0;
2185 }
2186
2187 static int
2188 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2189         struct rte_eth_rss_conf *rss_conf)
2190 {
2191         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2192         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2193         uint32_t mrqc;
2194         uint64_t hf = rss_conf->rss_hf;
2195         int i;
2196
2197         PMD_INIT_FUNC_TRACE();
2198
2199         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2200                                 FM10K_RSSRK_ENTRIES_PER_REG))
2201                 return -EINVAL;
2202
2203         if (hf == 0)
2204                 return -EINVAL;
2205
2206         mrqc = 0;
2207         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2208         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2209         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2210         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2211         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2212         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2213         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2214         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2215         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2216
2217         /* If the mapping doesn't fit any supported, return */
2218         if (mrqc == 0)
2219                 return -EINVAL;
2220
2221         if (key != NULL)
2222                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2223                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2224
2225         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2226
2227         return 0;
2228 }
2229
2230 static int
2231 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2232         struct rte_eth_rss_conf *rss_conf)
2233 {
2234         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2235         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2236         uint32_t mrqc;
2237         uint64_t hf;
2238         int i;
2239
2240         PMD_INIT_FUNC_TRACE();
2241
2242         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2243                                 FM10K_RSSRK_ENTRIES_PER_REG))
2244                 return -EINVAL;
2245
2246         if (key != NULL)
2247                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2248                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2249
2250         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2251         hf = 0;
2252         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2253         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2254         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2255         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2256         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2257         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2258         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2259         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2260         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2261
2262         rss_conf->rss_hf = hf;
2263
2264         return 0;
2265 }
2266
2267 static void
2268 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2269 {
2270         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2271         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2272
2273         /* Bind all local non-queue interrupt to vector 0 */
2274         int_map |= FM10K_MISC_VEC_ID;
2275
2276         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2277         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2278         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2279         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2280         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2281         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2282
2283         /* Enable misc causes */
2284         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2285                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2286                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2287                                 FM10K_EIMR_ENABLE(MAILBOX) |
2288                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2289                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2290                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2291                                 FM10K_EIMR_ENABLE(VFLR));
2292
2293         /* Enable ITR 0 */
2294         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2295                                         FM10K_ITR_MASK_CLEAR);
2296         FM10K_WRITE_FLUSH(hw);
2297 }
2298
2299 static void
2300 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2301 {
2302         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2303         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2304
2305         int_map |= FM10K_MISC_VEC_ID;
2306
2307         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2308         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2309         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2310         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2311         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2312         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2313
2314         /* Disable misc causes */
2315         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2316                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2317                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2318                                 FM10K_EIMR_DISABLE(MAILBOX) |
2319                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2320                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2321                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2322                                 FM10K_EIMR_DISABLE(VFLR));
2323
2324         /* Disable ITR 0 */
2325         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2326         FM10K_WRITE_FLUSH(hw);
2327 }
2328
2329 static void
2330 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2331 {
2332         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2333         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2334
2335         /* Bind all local non-queue interrupt to vector 0 */
2336         int_map |= FM10K_MISC_VEC_ID;
2337
2338         /* Only INT 0 available, other 15 are reserved. */
2339         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2340
2341         /* Enable ITR 0 */
2342         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2343                                         FM10K_ITR_MASK_CLEAR);
2344         FM10K_WRITE_FLUSH(hw);
2345 }
2346
2347 static void
2348 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2349 {
2350         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2351         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2352
2353         int_map |= FM10K_MISC_VEC_ID;
2354
2355         /* Only INT 0 available, other 15 are reserved. */
2356         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2357
2358         /* Disable ITR 0 */
2359         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2360         FM10K_WRITE_FLUSH(hw);
2361 }
2362
2363 static int
2364 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2365 {
2366         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2367         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2368
2369         /* Enable ITR */
2370         if (hw->mac.type == fm10k_mac_pf)
2371                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2372                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2373         else
2374                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2375                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2376         rte_intr_enable(&pdev->intr_handle);
2377         return 0;
2378 }
2379
2380 static int
2381 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2382 {
2383         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2384         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2385
2386         /* Disable ITR */
2387         if (hw->mac.type == fm10k_mac_pf)
2388                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2389                         FM10K_ITR_MASK_SET);
2390         else
2391                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2392                         FM10K_ITR_MASK_SET);
2393         return 0;
2394 }
2395
2396 static int
2397 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2398 {
2399         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2400         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2401         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2402         uint32_t intr_vector, vec;
2403         uint16_t queue_id;
2404         int result = 0;
2405
2406         /* fm10k needs one separate interrupt for mailbox,
2407          * so only drivers which support multiple interrupt vectors
2408          * e.g. vfio-pci can work for fm10k interrupt mode
2409          */
2410         if (!rte_intr_cap_multiple(intr_handle) ||
2411                         dev->data->dev_conf.intr_conf.rxq == 0)
2412                 return result;
2413
2414         intr_vector = dev->data->nb_rx_queues;
2415
2416         /* disable interrupt first */
2417         rte_intr_disable(intr_handle);
2418         if (hw->mac.type == fm10k_mac_pf)
2419                 fm10k_dev_disable_intr_pf(dev);
2420         else
2421                 fm10k_dev_disable_intr_vf(dev);
2422
2423         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2424                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2425                 result = -EIO;
2426         }
2427
2428         if (rte_intr_dp_is_en(intr_handle) && !result) {
2429                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2430                         dev->data->nb_rx_queues * sizeof(int), 0);
2431                 if (intr_handle->intr_vec) {
2432                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2433                                         queue_id < dev->data->nb_rx_queues;
2434                                         queue_id++) {
2435                                 intr_handle->intr_vec[queue_id] = vec;
2436                                 if (vec < intr_handle->nb_efd - 1
2437                                                 + FM10K_RX_VEC_START)
2438                                         vec++;
2439                         }
2440                 } else {
2441                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2442                                 " intr_vec", dev->data->nb_rx_queues);
2443                         rte_intr_efd_disable(intr_handle);
2444                         result = -ENOMEM;
2445                 }
2446         }
2447
2448         if (hw->mac.type == fm10k_mac_pf)
2449                 fm10k_dev_enable_intr_pf(dev);
2450         else
2451                 fm10k_dev_enable_intr_vf(dev);
2452         rte_intr_enable(intr_handle);
2453         hw->mac.ops.update_int_moderator(hw);
2454         return result;
2455 }
2456
2457 static int
2458 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2459 {
2460         struct fm10k_fault fault;
2461         int err;
2462         const char *estr = "Unknown error";
2463
2464         /* Process PCA fault */
2465         if (eicr & FM10K_EICR_PCA_FAULT) {
2466                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2467                 if (err)
2468                         goto error;
2469                 switch (fault.type) {
2470                 case PCA_NO_FAULT:
2471                         estr = "PCA_NO_FAULT"; break;
2472                 case PCA_UNMAPPED_ADDR:
2473                         estr = "PCA_UNMAPPED_ADDR"; break;
2474                 case PCA_BAD_QACCESS_PF:
2475                         estr = "PCA_BAD_QACCESS_PF"; break;
2476                 case PCA_BAD_QACCESS_VF:
2477                         estr = "PCA_BAD_QACCESS_VF"; break;
2478                 case PCA_MALICIOUS_REQ:
2479                         estr = "PCA_MALICIOUS_REQ"; break;
2480                 case PCA_POISONED_TLP:
2481                         estr = "PCA_POISONED_TLP"; break;
2482                 case PCA_TLP_ABORT:
2483                         estr = "PCA_TLP_ABORT"; break;
2484                 default:
2485                         goto error;
2486                 }
2487                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2488                         estr, fault.func ? "VF" : "PF", fault.func,
2489                         fault.address, fault.specinfo);
2490         }
2491
2492         /* Process THI fault */
2493         if (eicr & FM10K_EICR_THI_FAULT) {
2494                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2495                 if (err)
2496                         goto error;
2497                 switch (fault.type) {
2498                 case THI_NO_FAULT:
2499                         estr = "THI_NO_FAULT"; break;
2500                 case THI_MAL_DIS_Q_FAULT:
2501                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2502                 default:
2503                         goto error;
2504                 }
2505                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2506                         estr, fault.func ? "VF" : "PF", fault.func,
2507                         fault.address, fault.specinfo);
2508         }
2509
2510         /* Process FUM fault */
2511         if (eicr & FM10K_EICR_FUM_FAULT) {
2512                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2513                 if (err)
2514                         goto error;
2515                 switch (fault.type) {
2516                 case FUM_NO_FAULT:
2517                         estr = "FUM_NO_FAULT"; break;
2518                 case FUM_UNMAPPED_ADDR:
2519                         estr = "FUM_UNMAPPED_ADDR"; break;
2520                 case FUM_POISONED_TLP:
2521                         estr = "FUM_POISONED_TLP"; break;
2522                 case FUM_BAD_VF_QACCESS:
2523                         estr = "FUM_BAD_VF_QACCESS"; break;
2524                 case FUM_ADD_DECODE_ERR:
2525                         estr = "FUM_ADD_DECODE_ERR"; break;
2526                 case FUM_RO_ERROR:
2527                         estr = "FUM_RO_ERROR"; break;
2528                 case FUM_QPRC_CRC_ERROR:
2529                         estr = "FUM_QPRC_CRC_ERROR"; break;
2530                 case FUM_CSR_TIMEOUT:
2531                         estr = "FUM_CSR_TIMEOUT"; break;
2532                 case FUM_INVALID_TYPE:
2533                         estr = "FUM_INVALID_TYPE"; break;
2534                 case FUM_INVALID_LENGTH:
2535                         estr = "FUM_INVALID_LENGTH"; break;
2536                 case FUM_INVALID_BE:
2537                         estr = "FUM_INVALID_BE"; break;
2538                 case FUM_INVALID_ALIGN:
2539                         estr = "FUM_INVALID_ALIGN"; break;
2540                 default:
2541                         goto error;
2542                 }
2543                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2544                         estr, fault.func ? "VF" : "PF", fault.func,
2545                         fault.address, fault.specinfo);
2546         }
2547
2548         return 0;
2549 error:
2550         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2551         return err;
2552 }
2553
2554 /**
2555  * PF interrupt handler triggered by NIC for handling specific interrupt.
2556  *
2557  * @param handle
2558  *  Pointer to interrupt handle.
2559  * @param param
2560  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2561  *
2562  * @return
2563  *  void
2564  */
2565 static void
2566 fm10k_dev_interrupt_handler_pf(void *param)
2567 {
2568         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2569         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2570         uint32_t cause, status;
2571         struct fm10k_dev_info *dev_info =
2572                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2573         int status_mbx;
2574         s32 err;
2575
2576         if (hw->mac.type != fm10k_mac_pf)
2577                 return;
2578
2579         cause = FM10K_READ_REG(hw, FM10K_EICR);
2580
2581         /* Handle PCI fault cases */
2582         if (cause & FM10K_EICR_FAULT_MASK) {
2583                 PMD_INIT_LOG(ERR, "INT: find fault!");
2584                 fm10k_dev_handle_fault(hw, cause);
2585         }
2586
2587         /* Handle switch up/down */
2588         if (cause & FM10K_EICR_SWITCHNOTREADY)
2589                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2590
2591         if (cause & FM10K_EICR_SWITCHREADY) {
2592                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2593                 if (dev_info->sm_down == 1) {
2594                         fm10k_mbx_lock(hw);
2595
2596                         /* For recreating logical ports */
2597                         status_mbx = hw->mac.ops.update_lport_state(hw,
2598                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2599                         if (status_mbx == FM10K_SUCCESS)
2600                                 PMD_INIT_LOG(INFO,
2601                                         "INT: Recreated Logical port");
2602                         else
2603                                 PMD_INIT_LOG(INFO,
2604                                         "INT: Logical ports weren't recreated");
2605
2606                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2607                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2608                         if (status_mbx != FM10K_SUCCESS)
2609                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2610
2611                         fm10k_mbx_unlock(hw);
2612
2613                         /* first clear the internal SW recording structure */
2614                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2615                                                 ETH_MQ_RX_VMDQ_FLAG))
2616                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2617                                         false);
2618
2619                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2620                                         MAIN_VSI_POOL_NUMBER);
2621
2622                         /*
2623                          * Add default mac address and vlan for the logical
2624                          * ports that have been created, leave to the
2625                          * application to fully recover Rx filtering.
2626                          */
2627                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2628                                         MAIN_VSI_POOL_NUMBER);
2629
2630                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2631                                                 ETH_MQ_RX_VMDQ_FLAG))
2632                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2633                                         true);
2634
2635                         dev_info->sm_down = 0;
2636                         _rte_eth_dev_callback_process(dev,
2637                                         RTE_ETH_EVENT_INTR_LSC,
2638                                         NULL);
2639                 }
2640         }
2641
2642         /* Handle mailbox message */
2643         fm10k_mbx_lock(hw);
2644         err = hw->mbx.ops.process(hw, &hw->mbx);
2645         fm10k_mbx_unlock(hw);
2646
2647         if (err == FM10K_ERR_RESET_REQUESTED) {
2648                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2649                 dev_info->sm_down = 1;
2650                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2651                                 NULL);
2652         }
2653
2654         /* Handle SRAM error */
2655         if (cause & FM10K_EICR_SRAMERROR) {
2656                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2657
2658                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2659                 /* Write to clear pending bits */
2660                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2661
2662                 /* Todo: print out error message after shared code  updates */
2663         }
2664
2665         /* Clear these 3 events if having any */
2666         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2667                  FM10K_EICR_SWITCHREADY;
2668         if (cause)
2669                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2670
2671         /* Re-enable interrupt from device side */
2672         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2673                                         FM10K_ITR_MASK_CLEAR);
2674         /* Re-enable interrupt from host side */
2675         rte_intr_enable(dev->intr_handle);
2676 }
2677
2678 /**
2679  * VF interrupt handler triggered by NIC for handling specific interrupt.
2680  *
2681  * @param handle
2682  *  Pointer to interrupt handle.
2683  * @param param
2684  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2685  *
2686  * @return
2687  *  void
2688  */
2689 static void
2690 fm10k_dev_interrupt_handler_vf(void *param)
2691 {
2692         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2693         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2694         struct fm10k_mbx_info *mbx = &hw->mbx;
2695         struct fm10k_dev_info *dev_info =
2696                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2697         const enum fm10k_mbx_state state = mbx->state;
2698         int status_mbx;
2699
2700         if (hw->mac.type != fm10k_mac_vf)
2701                 return;
2702
2703         /* Handle mailbox message if lock is acquired */
2704         fm10k_mbx_lock(hw);
2705         hw->mbx.ops.process(hw, &hw->mbx);
2706         fm10k_mbx_unlock(hw);
2707
2708         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2709                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2710
2711                 fm10k_mbx_lock(hw);
2712                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2713                                 MAX_LPORT_NUM, 1);
2714                 fm10k_mbx_unlock(hw);
2715
2716                 /* Setting reset flag */
2717                 dev_info->sm_down = 1;
2718                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2719                                 NULL);
2720         }
2721
2722         if (dev_info->sm_down == 1 &&
2723                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2724                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2725                 fm10k_mbx_lock(hw);
2726                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2727                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2728                 if (status_mbx != FM10K_SUCCESS)
2729                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2730                 fm10k_mbx_unlock(hw);
2731
2732                 /* first clear the internal SW recording structure */
2733                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2734                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2735                                 MAIN_VSI_POOL_NUMBER);
2736
2737                 /*
2738                  * Add default mac address and vlan for the logical ports that
2739                  * have been created, leave to the application to fully recover
2740                  * Rx filtering.
2741                  */
2742                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2743                                 MAIN_VSI_POOL_NUMBER);
2744                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2745
2746                 dev_info->sm_down = 0;
2747                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2748                                 NULL);
2749         }
2750
2751         /* Re-enable interrupt from device side */
2752         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2753                                         FM10K_ITR_MASK_CLEAR);
2754         /* Re-enable interrupt from host side */
2755         rte_intr_enable(dev->intr_handle);
2756 }
2757
2758 /* Mailbox message handler in VF */
2759 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2760         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2761         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2762         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2763         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2764 };
2765
2766 static int
2767 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2768 {
2769         int err = 0;
2770
2771         /* Initialize mailbox lock */
2772         fm10k_mbx_initlock(hw);
2773
2774         /* Replace default message handler with new ones */
2775         if (hw->mac.type == fm10k_mac_vf)
2776                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2777
2778         if (err) {
2779                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2780                                 err);
2781                 return err;
2782         }
2783         /* Connect to SM for PF device or PF for VF device */
2784         return hw->mbx.ops.connect(hw, &hw->mbx);
2785 }
2786
2787 static void
2788 fm10k_close_mbx_service(struct fm10k_hw *hw)
2789 {
2790         /* Disconnect from SM for PF device or PF for VF device */
2791         hw->mbx.ops.disconnect(hw, &hw->mbx);
2792 }
2793
2794 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2795         .dev_configure          = fm10k_dev_configure,
2796         .dev_start              = fm10k_dev_start,
2797         .dev_stop               = fm10k_dev_stop,
2798         .dev_close              = fm10k_dev_close,
2799         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2800         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2801         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2802         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2803         .stats_get              = fm10k_stats_get,
2804         .xstats_get             = fm10k_xstats_get,
2805         .xstats_get_names       = fm10k_xstats_get_names,
2806         .stats_reset            = fm10k_stats_reset,
2807         .xstats_reset           = fm10k_stats_reset,
2808         .link_update            = fm10k_link_update,
2809         .dev_infos_get          = fm10k_dev_infos_get,
2810         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2811         .vlan_filter_set        = fm10k_vlan_filter_set,
2812         .vlan_offload_set       = fm10k_vlan_offload_set,
2813         .mac_addr_add           = fm10k_macaddr_add,
2814         .mac_addr_remove        = fm10k_macaddr_remove,
2815         .rx_queue_start         = fm10k_dev_rx_queue_start,
2816         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2817         .tx_queue_start         = fm10k_dev_tx_queue_start,
2818         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2819         .rx_queue_setup         = fm10k_rx_queue_setup,
2820         .rx_queue_release       = fm10k_rx_queue_release,
2821         .tx_queue_setup         = fm10k_tx_queue_setup,
2822         .tx_queue_release       = fm10k_tx_queue_release,
2823         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2824         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2825         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2826         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2827         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2828         .reta_update            = fm10k_reta_update,
2829         .reta_query             = fm10k_reta_query,
2830         .rss_hash_update        = fm10k_rss_hash_update,
2831         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2832 };
2833
2834 static int ftag_check_handler(__rte_unused const char *key,
2835                 const char *value, __rte_unused void *opaque)
2836 {
2837         if (strcmp(value, "1"))
2838                 return -1;
2839
2840         return 0;
2841 }
2842
2843 static int
2844 fm10k_check_ftag(struct rte_devargs *devargs)
2845 {
2846         struct rte_kvargs *kvlist;
2847         const char *ftag_key = "enable_ftag";
2848
2849         if (devargs == NULL)
2850                 return 0;
2851
2852         kvlist = rte_kvargs_parse(devargs->args, NULL);
2853         if (kvlist == NULL)
2854                 return 0;
2855
2856         if (!rte_kvargs_count(kvlist, ftag_key)) {
2857                 rte_kvargs_free(kvlist);
2858                 return 0;
2859         }
2860         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2861         if (rte_kvargs_process(kvlist, ftag_key,
2862                                 ftag_check_handler, NULL) < 0) {
2863                 rte_kvargs_free(kvlist);
2864                 return 0;
2865         }
2866         rte_kvargs_free(kvlist);
2867
2868         return 1;
2869 }
2870
2871 static uint16_t
2872 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2873                     uint16_t nb_pkts)
2874 {
2875         uint16_t nb_tx = 0;
2876         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2877
2878         while (nb_pkts) {
2879                 uint16_t ret, num;
2880
2881                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2882                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2883                                                  num);
2884                 nb_tx += ret;
2885                 nb_pkts -= ret;
2886                 if (ret < num)
2887                         break;
2888         }
2889
2890         return nb_tx;
2891 }
2892
2893 static void __attribute__((cold))
2894 fm10k_set_tx_function(struct rte_eth_dev *dev)
2895 {
2896         struct fm10k_tx_queue *txq;
2897         int i;
2898         int use_sse = 1;
2899         uint16_t tx_ftag_en = 0;
2900
2901         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2902                 /* primary process has set the ftag flag and offloads */
2903                 txq = dev->data->tx_queues[0];
2904                 if (fm10k_tx_vec_condition_check(txq)) {
2905                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2906                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2907                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2908                 } else {
2909                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2910                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2911                         dev->tx_pkt_prepare = NULL;
2912                 }
2913                 return;
2914         }
2915
2916         if (fm10k_check_ftag(dev->device->devargs))
2917                 tx_ftag_en = 1;
2918
2919         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2920                 txq = dev->data->tx_queues[i];
2921                 txq->tx_ftag_en = tx_ftag_en;
2922                 /* Check if Vector Tx is satisfied */
2923                 if (fm10k_tx_vec_condition_check(txq))
2924                         use_sse = 0;
2925         }
2926
2927         if (use_sse) {
2928                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2929                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2930                         txq = dev->data->tx_queues[i];
2931                         fm10k_txq_vec_setup(txq);
2932                 }
2933                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2934                 dev->tx_pkt_prepare = NULL;
2935         } else {
2936                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2937                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2938                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2939         }
2940 }
2941
2942 static void __attribute__((cold))
2943 fm10k_set_rx_function(struct rte_eth_dev *dev)
2944 {
2945         struct fm10k_dev_info *dev_info =
2946                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2947         uint16_t i, rx_using_sse;
2948         uint16_t rx_ftag_en = 0;
2949
2950         if (fm10k_check_ftag(dev->device->devargs))
2951                 rx_ftag_en = 1;
2952
2953         /* In order to allow Vector Rx there are a few configuration
2954          * conditions to be met.
2955          */
2956         if (!fm10k_rx_vec_condition_check(dev) &&
2957                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2958                 if (dev->data->scattered_rx)
2959                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2960                 else
2961                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2962         } else if (dev->data->scattered_rx)
2963                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2964         else
2965                 dev->rx_pkt_burst = fm10k_recv_pkts;
2966
2967         rx_using_sse =
2968                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2969                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2970
2971         if (rx_using_sse)
2972                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2973         else
2974                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2975
2976         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2977                 return;
2978
2979         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2980                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2981
2982                 rxq->rx_using_sse = rx_using_sse;
2983                 rxq->rx_ftag_en = rx_ftag_en;
2984         }
2985 }
2986
2987 static void
2988 fm10k_params_init(struct rte_eth_dev *dev)
2989 {
2990         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2991         struct fm10k_dev_info *info =
2992                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2993
2994         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2995          * there is no way to get link status without reading BAR4.  Until this
2996          * works, assume we have maximum bandwidth.
2997          * @todo - fix bus info
2998          */
2999         hw->bus_caps.speed = fm10k_bus_speed_8000;
3000         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3001         hw->bus_caps.payload = fm10k_bus_payload_512;
3002         hw->bus.speed = fm10k_bus_speed_8000;
3003         hw->bus.width = fm10k_bus_width_pcie_x8;
3004         hw->bus.payload = fm10k_bus_payload_256;
3005
3006         info->rx_vec_allowed = true;
3007 }
3008
3009 static int
3010 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3011 {
3012         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3013         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3014         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3015         int diag, i;
3016         struct fm10k_macvlan_filter_info *macvlan;
3017
3018         PMD_INIT_FUNC_TRACE();
3019
3020         dev->dev_ops = &fm10k_eth_dev_ops;
3021         dev->rx_pkt_burst = &fm10k_recv_pkts;
3022         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3023         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3024
3025         /*
3026          * Primary process does the whole initialization, for secondary
3027          * processes, we just select the same Rx and Tx function as primary.
3028          */
3029         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3030                 fm10k_set_rx_function(dev);
3031                 fm10k_set_tx_function(dev);
3032                 return 0;
3033         }
3034
3035         rte_eth_copy_pci_info(dev, pdev);
3036
3037         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3038         memset(macvlan, 0, sizeof(*macvlan));
3039         /* Vendor and Device ID need to be set before init of shared code */
3040         memset(hw, 0, sizeof(*hw));
3041         hw->device_id = pdev->id.device_id;
3042         hw->vendor_id = pdev->id.vendor_id;
3043         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3044         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3045         hw->revision_id = 0;
3046         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3047         if (hw->hw_addr == NULL) {
3048                 PMD_INIT_LOG(ERR, "Bad mem resource."
3049                         " Try to blacklist unused devices.");
3050                 return -EIO;
3051         }
3052
3053         /* Store fm10k_adapter pointer */
3054         hw->back = dev->data->dev_private;
3055
3056         /* Initialize the shared code */
3057         diag = fm10k_init_shared_code(hw);
3058         if (diag != FM10K_SUCCESS) {
3059                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3060                 return -EIO;
3061         }
3062
3063         /* Initialize parameters */
3064         fm10k_params_init(dev);
3065
3066         /* Initialize the hw */
3067         diag = fm10k_init_hw(hw);
3068         if (diag != FM10K_SUCCESS) {
3069                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3070                 return -EIO;
3071         }
3072
3073         /* Initialize MAC address(es) */
3074         dev->data->mac_addrs = rte_zmalloc("fm10k",
3075                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3076         if (dev->data->mac_addrs == NULL) {
3077                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3078                 return -ENOMEM;
3079         }
3080
3081         diag = fm10k_read_mac_addr(hw);
3082
3083         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3084                         &dev->data->mac_addrs[0]);
3085
3086         if (diag != FM10K_SUCCESS ||
3087                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3088
3089                 /* Generate a random addr */
3090                 eth_random_addr(hw->mac.addr);
3091                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3092                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3093                 &dev->data->mac_addrs[0]);
3094         }
3095
3096         /* Reset the hw statistics */
3097         fm10k_stats_reset(dev);
3098
3099         /* Reset the hw */
3100         diag = fm10k_reset_hw(hw);
3101         if (diag != FM10K_SUCCESS) {
3102                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3103                 return -EIO;
3104         }
3105
3106         /* Setup mailbox service */
3107         diag = fm10k_setup_mbx_service(hw);
3108         if (diag != FM10K_SUCCESS) {
3109                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3110                 return -EIO;
3111         }
3112
3113         /*PF/VF has different interrupt handling mechanism */
3114         if (hw->mac.type == fm10k_mac_pf) {
3115                 /* register callback func to eal lib */
3116                 rte_intr_callback_register(intr_handle,
3117                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3118
3119                 /* enable MISC interrupt */
3120                 fm10k_dev_enable_intr_pf(dev);
3121         } else { /* VF */
3122                 rte_intr_callback_register(intr_handle,
3123                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3124
3125                 fm10k_dev_enable_intr_vf(dev);
3126         }
3127
3128         /* Enable intr after callback registered */
3129         rte_intr_enable(intr_handle);
3130
3131         hw->mac.ops.update_int_moderator(hw);
3132
3133         /* Make sure Switch Manager is ready before going forward. */
3134         if (hw->mac.type == fm10k_mac_pf) {
3135                 int switch_ready = 0;
3136
3137                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3138                         fm10k_mbx_lock(hw);
3139                         hw->mac.ops.get_host_state(hw, &switch_ready);
3140                         fm10k_mbx_unlock(hw);
3141                         if (switch_ready)
3142                                 break;
3143                         /* Delay some time to acquire async LPORT_MAP info. */
3144                         rte_delay_us(WAIT_SWITCH_MSG_US);
3145                 }
3146
3147                 if (switch_ready == 0) {
3148                         PMD_INIT_LOG(ERR, "switch is not ready");
3149                         return -1;
3150                 }
3151         }
3152
3153         /*
3154          * Below function will trigger operations on mailbox, acquire lock to
3155          * avoid race condition from interrupt handler. Operations on mailbox
3156          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3157          * will handle and generate an interrupt to our side. Then,  FIFO in
3158          * mailbox will be touched.
3159          */
3160         fm10k_mbx_lock(hw);
3161         /* Enable port first */
3162         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3163                                         MAX_LPORT_NUM, 1);
3164
3165         /* Set unicast mode by default. App can change to other mode in other
3166          * API func.
3167          */
3168         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3169                                         FM10K_XCAST_MODE_NONE);
3170
3171         fm10k_mbx_unlock(hw);
3172
3173         /* Make sure default VID is ready before going forward. */
3174         if (hw->mac.type == fm10k_mac_pf) {
3175                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3176                         if (hw->mac.default_vid)
3177                                 break;
3178                         /* Delay some time to acquire async port VLAN info. */
3179                         rte_delay_us(WAIT_SWITCH_MSG_US);
3180                 }
3181
3182                 if (!hw->mac.default_vid) {
3183                         PMD_INIT_LOG(ERR, "default VID is not ready");
3184                         return -1;
3185                 }
3186         }
3187
3188         /* Add default mac address */
3189         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3190                 MAIN_VSI_POOL_NUMBER);
3191
3192         return 0;
3193 }
3194
3195 static int
3196 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3197 {
3198         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3199         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3200         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3201         PMD_INIT_FUNC_TRACE();
3202
3203         /* only uninitialize in the primary process */
3204         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3205                 return 0;
3206
3207         /* safe to close dev here */
3208         fm10k_dev_close(dev);
3209
3210         dev->dev_ops = NULL;
3211         dev->rx_pkt_burst = NULL;
3212         dev->tx_pkt_burst = NULL;
3213
3214         /* disable uio/vfio intr */
3215         rte_intr_disable(intr_handle);
3216
3217         /*PF/VF has different interrupt handling mechanism */
3218         if (hw->mac.type == fm10k_mac_pf) {
3219                 /* disable interrupt */
3220                 fm10k_dev_disable_intr_pf(dev);
3221
3222                 /* unregister callback func to eal lib */
3223                 rte_intr_callback_unregister(intr_handle,
3224                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3225         } else {
3226                 /* disable interrupt */
3227                 fm10k_dev_disable_intr_vf(dev);
3228
3229                 rte_intr_callback_unregister(intr_handle,
3230                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3231         }
3232
3233         /* free mac memory */
3234         if (dev->data->mac_addrs) {
3235                 rte_free(dev->data->mac_addrs);
3236                 dev->data->mac_addrs = NULL;
3237         }
3238
3239         memset(hw, 0, sizeof(*hw));
3240
3241         return 0;
3242 }
3243
3244 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3245         struct rte_pci_device *pci_dev)
3246 {
3247         return rte_eth_dev_pci_generic_probe(pci_dev,
3248                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3249 }
3250
3251 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3252 {
3253         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3254 }
3255
3256 /*
3257  * The set of PCI devices this driver supports. This driver will enable both PF
3258  * and SRIOV-VF devices.
3259  */
3260 static const struct rte_pci_id pci_id_fm10k_map[] = {
3261         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3262         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3263         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3264         { .vendor_id = 0, /* sentinel */ },
3265 };
3266
3267 static struct rte_pci_driver rte_pmd_fm10k = {
3268         .id_table = pci_id_fm10k_map,
3269         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3270                      RTE_PCI_DRV_IOVA_AS_VA,
3271         .probe = eth_fm10k_pci_probe,
3272         .remove = eth_fm10k_pci_remove,
3273 };
3274
3275 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3276 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3277 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3278
3279 RTE_INIT(fm10k_init_log)
3280 {
3281         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3282         if (fm10k_logtype_init >= 0)
3283                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3284         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3285         if (fm10k_logtype_driver >= 0)
3286                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3287 }