drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* KEEP_CRC offload flag is not supported by PMD
 455          * can remove the below block when DEV_RX_OFFLOAD_CRC_STRIP removed
 456          */
 457         if (rte_eth_dev_must_keep_crc(dev->data->dev_conf.rxmode.offloads))
 458                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 459
 460         /* multipe queue mode checking */
 461         ret  = fm10k_check_mq_mode(dev);
 462         if (ret != 0) {
 463                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 464                             ret);
 465                 return ret;
 466         }
 467
 468         dev->data->scattered_rx = 0;
 469
 470         return 0;
 471 }
 472
 473 /* fls = find last set bit = 32 minus the number of leading zeros */
 474 #ifndef fls
 475 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 476 #endif
 477
 478 static void
 479 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 480 {
 481         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 482         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 483         uint32_t i;
 484
 485         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 486
 487         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 488                 if (!vmdq_conf->pool_map[i].pools)
 489                         continue;
 490                 fm10k_mbx_lock(hw);
 491                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 492                 fm10k_mbx_unlock(hw);
 493         }
 494 }
 495
 496 static void
 497 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 498 {
 499         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 500
 501         /* Add default mac address */
 502         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 503                 MAIN_VSI_POOL_NUMBER);
 504 }
 505
 506 static void
 507 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 508 {
 509         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 510         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 511         uint32_t mrqc, *key, i, reta, j;
 512         uint64_t hf;
 513
 514 #define RSS_KEY_SIZE 40
 515         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 516                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 517                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 518                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 519                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 520                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 521         };
 522
 523         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 524                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 525                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 526                 return;
 527         }
 528
 529         /* random key is rss_intel_key (default) or user provided (rss_key) */
 530         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 531                 key = (uint32_t *)rss_intel_key;
 532         else
 533                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 534
 535         /* Now fill our hash function seeds, 4 bytes at a time */
 536         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 537                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 538
 539         /*
 540          * Fill in redirection table
 541          * The byte-swap is needed because NIC registers are in
 542          * little-endian order.
 543          */
 544         reta = 0;
 545         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 546                 if (j == dev->data->nb_rx_queues)
 547                         j = 0;
 548                 reta = (reta << CHAR_BIT) | j;
 549                 if ((i & 3) == 3)
 550                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 551                                         rte_bswap32(reta));
 552         }
 553
 554         /*
 555          * Generate RSS hash based on packet types, TCP/UDP
 556          * port numbers and/or IPv4/v6 src and dst addresses
 557          */
 558         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 559         mrqc = 0;
 560         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 561         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 562         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 563         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 564         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 565         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 566         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 567         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 568         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 569
 570         if (mrqc == 0) {
 571                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 572                         "supported", hf);
 573                 return;
 574         }
 575
 576         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 577 }
 578
 579 static void
 580 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 581 {
 582         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 583         uint32_t i;
 584
 585         for (i = 0; i < nb_lport_new; i++) {
 586                 /* Set unicast mode by default. App can change
 587                  * to other mode in other API func.
 588                  */
 589                 fm10k_mbx_lock(hw);
 590                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 591                         FM10K_XCAST_MODE_NONE);
 592                 fm10k_mbx_unlock(hw);
 593         }
 594 }
 595
 596 static void
 597 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 598 {
 599         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 600         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 601         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 602         struct fm10k_macvlan_filter_info *macvlan;
 603         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 604         uint16_t nb_lport_new;
 605
 606         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 607         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 608
 609         fm10k_dev_rss_configure(dev);
 610
 611         /* only PF supports VMDQ */
 612         if (hw->mac.type != fm10k_mac_pf)
 613                 return;
 614
 615         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 616                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 617
 618         /* no pool number change, no need to update logic port and VLAN/MAC */
 619         if (macvlan->nb_queue_pools == nb_queue_pools)
 620                 return;
 621
 622         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 623         fm10k_dev_logic_port_update(dev, nb_lport_new);
 624
 625         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 626         memset(dev->data->mac_addrs, 0,
 627                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 628         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 629                 &dev->data->mac_addrs[0]);
 630         memset(macvlan, 0, sizeof(*macvlan));
 631         macvlan->nb_queue_pools = nb_queue_pools;
 632
 633         if (nb_queue_pools)
 634                 fm10k_dev_vmdq_rx_configure(dev);
 635         else
 636                 fm10k_dev_pf_main_vsi_reset(dev);
 637 }
 638
 639 static int
 640 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 641 {
 642         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 643         int i, ret;
 644         struct fm10k_tx_queue *txq;
 645         uint64_t base_addr;
 646         uint32_t size;
 647
 648         /* Disable TXINT to avoid possible interrupt */
 649         for (i = 0; i < hw->mac.max_queues; i++)
 650                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 651                                 3 << FM10K_TXINT_TIMER_SHIFT);
 652
 653         /* Setup TX queue */
 654         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 655                 txq = dev->data->tx_queues[i];
 656                 base_addr = txq->hw_ring_phys_addr;
 657                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 658
 659                 /* disable queue to avoid issues while updating state */
 660                 ret = tx_queue_disable(hw, i);
 661                 if (ret) {
 662                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 663                         return -1;
 664                 }
 665                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 666                  * register is read-only for VF.
 667                  */
 668                 if (fm10k_check_ftag(dev->device->devargs)) {
 669                         if (hw->mac.type == fm10k_mac_pf) {
 670                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 671                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 672                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 673                         } else {
 674                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 675                                 return -ENOTSUP;
 676                         }
 677                 }
 678
 679                 /* set location and size for descriptor ring */
 680                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 681                                 base_addr & UINT64_LOWER_32BITS_MASK);
 682                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 683                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 684                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 685
 686                 /* assign default SGLORT for each TX queue by PF */
 687                 if (hw->mac.type == fm10k_mac_pf)
 688                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 689         }
 690
 691         /* set up vector or scalar TX function as appropriate */
 692         fm10k_set_tx_function(dev);
 693
 694         return 0;
 695 }
 696
 697 static int
 698 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 699 {
 700         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 701         struct fm10k_macvlan_filter_info *macvlan;
 702         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 703         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 704         int i, ret;
 705         struct fm10k_rx_queue *rxq;
 706         uint64_t base_addr;
 707         uint32_t size;
 708         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 709         uint32_t logic_port = hw->mac.dglort_map;
 710         uint16_t buf_size;
 711         uint16_t queue_stride = 0;
 712
 713         /* enable RXINT for interrupt mode */
 714         i = 0;
 715         if (rte_intr_dp_is_en(intr_handle)) {
 716                 for (; i < dev->data->nb_rx_queues; i++) {
 717                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 718                         if (hw->mac.type == fm10k_mac_pf)
 719                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 720                                         FM10K_ITR_AUTOMASK |
 721                                         FM10K_ITR_MASK_CLEAR);
 722                         else
 723                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 724                                         FM10K_ITR_AUTOMASK |
 725                                         FM10K_ITR_MASK_CLEAR);
 726                 }
 727         }
 728         /* Disable other RXINT to avoid possible interrupt */
 729         for (; i < hw->mac.max_queues; i++)
 730                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 731                         3 << FM10K_RXINT_TIMER_SHIFT);
 732
 733         /* Setup RX queues */
 734         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 735                 rxq = dev->data->rx_queues[i];
 736                 base_addr = rxq->hw_ring_phys_addr;
 737                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 738
 739                 /* disable queue to avoid issues while updating state */
 740                 ret = rx_queue_disable(hw, i);
 741                 if (ret) {
 742                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 743                         return -1;
 744                 }
 745
 746                 /* Setup the Base and Length of the Rx Descriptor Ring */
 747                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 748                                 base_addr & UINT64_LOWER_32BITS_MASK);
 749                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 750                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 751                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 752
 753                 /* Configure the Rx buffer size for one buff without split */
 754                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 755                         RTE_PKTMBUF_HEADROOM);
 756                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 757                  * reserved for this purpose, and the worst case could be 511B.
 758                  * But SRR reg assumes all buffers have the same size. In order
 759                  * to fill the gap, we'll have to consider the worst case and
 760                  * assume 512B is reserved. If we don't do so, it's possible
 761                  * for HW to overwrite data to next mbuf.
 762                  */
 763                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 764
 765                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 766                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 767                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 768
 769                 /* It adds dual VLAN length for supporting dual VLAN */
 770                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 771                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 772                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 773                         uint32_t reg;
 774                         dev->data->scattered_rx = 1;
 775                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 776                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 777                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 778                 }
 779
 780                 /* Enable drop on empty, it's RO for VF */
 781                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 782                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 783
 784                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 785                 FM10K_WRITE_FLUSH(hw);
 786         }
 787
 788         /* Configure VMDQ/RSS if applicable */
 789         fm10k_dev_mq_rx_configure(dev);
 790
 791         /* Decide the best RX function */
 792         fm10k_set_rx_function(dev);
 793
 794         /* update RX_SGLORT for loopback suppress*/
 795         if (hw->mac.type != fm10k_mac_pf)
 796                 return 0;
 797         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 798         if (macvlan->nb_queue_pools)
 799                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 800         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 801                 if (i && queue_stride && !(i % queue_stride))
 802                         logic_port++;
 803                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 804         }
 805
 806         return 0;
 807 }
 808
 809 static int
 810 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 811 {
 812         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 813         int err = -1;
 814         uint32_t reg;
 815         struct fm10k_rx_queue *rxq;
 816
 817         PMD_INIT_FUNC_TRACE();
 818
 819         if (rx_queue_id < dev->data->nb_rx_queues) {
 820                 rxq = dev->data->rx_queues[rx_queue_id];
 821                 err = rx_queue_reset(rxq);
 822                 if (err == -ENOMEM) {
 823                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 824                         return err;
 825                 } else if (err == -EINVAL) {
 826                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 827                                 " %d", err);
 828                         return err;
 829                 }
 830
 831                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 832                  * Note: this must be done AFTER the queue is enabled on real
 833                  * hardware, but BEFORE the queue is enabled when using the
 834                  * emulation platform. Do it in both places for now and remove
 835                  * this comment and the following two register writes when the
 836                  * emulation platform is no longer being used.
 837                  */
 838                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 839                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 840
 841                 /* Set PF ownership flag for PF devices */
 842                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 843                 if (hw->mac.type == fm10k_mac_pf)
 844                         reg |= FM10K_RXQCTL_PF;
 845                 reg |= FM10K_RXQCTL_ENABLE;
 846                 /* enable RX queue */
 847                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 848                 FM10K_WRITE_FLUSH(hw);
 849
 850                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 851                  * Note: this must be done AFTER the queue is enabled
 852                  */
 853                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 854                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 855                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 856         }
 857
 858         return err;
 859 }
 860
 861 static int
 862 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 863 {
 864         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 865
 866         PMD_INIT_FUNC_TRACE();
 867
 868         if (rx_queue_id < dev->data->nb_rx_queues) {
 869                 /* Disable RX queue */
 870                 rx_queue_disable(hw, rx_queue_id);
 871
 872                 /* Free mbuf and clean HW ring */
 873                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 874                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 875         }
 876
 877         return 0;
 878 }
 879
 880 static int
 881 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 882 {
 883         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 884         /** @todo - this should be defined in the shared code */
 885 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 886         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 887         int err = 0;
 888
 889         PMD_INIT_FUNC_TRACE();
 890
 891         if (tx_queue_id < dev->data->nb_tx_queues) {
 892                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 893
 894                 q->ops->reset(q);
 895
 896                 /* reset head and tail pointers */
 897                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 898                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 899
 900                 /* enable TX queue */
 901                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 902                                         FM10K_TXDCTL_ENABLE | txdctl);
 903                 FM10K_WRITE_FLUSH(hw);
 904                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 905         } else
 906                 err = -1;
 907
 908         return err;
 909 }
 910
 911 static int
 912 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915
 916         PMD_INIT_FUNC_TRACE();
 917
 918         if (tx_queue_id < dev->data->nb_tx_queues) {
 919                 tx_queue_disable(hw, tx_queue_id);
 920                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 921                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 922         }
 923
 924         return 0;
 925 }
 926
 927 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 928 {
 929         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 930                 != FM10K_DGLORTMAP_NONE);
 931 }
 932
 933 static void
 934 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 935 {
 936         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 937         int status;
 938
 939         PMD_INIT_FUNC_TRACE();
 940
 941         /* Return if it didn't acquire valid glort range */
 942         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 943                 return;
 944
 945         fm10k_mbx_lock(hw);
 946         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 947                                 FM10K_XCAST_MODE_PROMISC);
 948         fm10k_mbx_unlock(hw);
 949
 950         if (status != FM10K_SUCCESS)
 951                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 952 }
 953
 954 static void
 955 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 956 {
 957         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 958         uint8_t mode;
 959         int status;
 960
 961         PMD_INIT_FUNC_TRACE();
 962
 963         /* Return if it didn't acquire valid glort range */
 964         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 965                 return;
 966
 967         if (dev->data->all_multicast == 1)
 968                 mode = FM10K_XCAST_MODE_ALLMULTI;
 969         else
 970                 mode = FM10K_XCAST_MODE_NONE;
 971
 972         fm10k_mbx_lock(hw);
 973         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 974                                 mode);
 975         fm10k_mbx_unlock(hw);
 976
 977         if (status != FM10K_SUCCESS)
 978                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 979 }
 980
 981 static void
 982 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 983 {
 984         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 985         int status;
 986
 987         PMD_INIT_FUNC_TRACE();
 988
 989         /* Return if it didn't acquire valid glort range */
 990         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 991                 return;
 992
 993         /* If promiscuous mode is enabled, it doesn't make sense to enable
 994          * allmulticast and disable promiscuous since fm10k only can select
 995          * one of the modes.
 996          */
 997         if (dev->data->promiscuous) {
 998                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 999                         "needn't enable allmulticast");
1000                 return;
1001         }
1002
1003         fm10k_mbx_lock(hw);
1004         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1005                                 FM10K_XCAST_MODE_ALLMULTI);
1006         fm10k_mbx_unlock(hw);
1007
1008         if (status != FM10K_SUCCESS)
1009                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1010 }
1011
1012 static void
1013 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1014 {
1015         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1016         int status;
1017
1018         PMD_INIT_FUNC_TRACE();
1019
1020         /* Return if it didn't acquire valid glort range */
1021         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1022                 return;
1023
1024         if (dev->data->promiscuous) {
1025                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1026                         "since promisc mode is enabled");
1027                 return;
1028         }
1029
1030         fm10k_mbx_lock(hw);
1031         /* Change mode to unicast mode */
1032         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1033                                 FM10K_XCAST_MODE_NONE);
1034         fm10k_mbx_unlock(hw);
1035
1036         if (status != FM10K_SUCCESS)
1037                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1038 }
1039
1040 static void
1041 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1042 {
1043         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1044         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1045         uint16_t nb_queue_pools;
1046         struct fm10k_macvlan_filter_info *macvlan;
1047
1048         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1049         nb_queue_pools = macvlan->nb_queue_pools;
1050         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1051         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1052
1053         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1054         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1055         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1056                         hw->mac.dglort_map;
1057         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1058         /* Configure VMDQ/RSS DGlort Decoder */
1059         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1060
1061         /* Flow Director configurations, only queue number is valid. */
1062         dglortdec = fls(dev->data->nb_rx_queues - 1);
1063         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1064                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1065         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1066         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1067
1068         /* Invalidate all other GLORT entries */
1069         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1070                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1071                                 FM10K_DGLORTMAP_NONE);
1072 }
1073
1074 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1075 static int
1076 fm10k_dev_start(struct rte_eth_dev *dev)
1077 {
1078         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1079         int i, diag;
1080
1081         PMD_INIT_FUNC_TRACE();
1082
1083         /* stop, init, then start the hw */
1084         diag = fm10k_stop_hw(hw);
1085         if (diag != FM10K_SUCCESS) {
1086                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1087                 return -EIO;
1088         }
1089
1090         diag = fm10k_init_hw(hw);
1091         if (diag != FM10K_SUCCESS) {
1092                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1093                 return -EIO;
1094         }
1095
1096         diag = fm10k_start_hw(hw);
1097         if (diag != FM10K_SUCCESS) {
1098                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1099                 return -EIO;
1100         }
1101
1102         diag = fm10k_dev_tx_init(dev);
1103         if (diag) {
1104                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1105                 return diag;
1106         }
1107
1108         if (fm10k_dev_rxq_interrupt_setup(dev))
1109                 return -EIO;
1110
1111         diag = fm10k_dev_rx_init(dev);
1112         if (diag) {
1113                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1114                 return diag;
1115         }
1116
1117         if (hw->mac.type == fm10k_mac_pf)
1118                 fm10k_dev_dglort_map_configure(dev);
1119
1120         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1121                 struct fm10k_rx_queue *rxq;
1122                 rxq = dev->data->rx_queues[i];
1123
1124                 if (rxq->rx_deferred_start)
1125                         continue;
1126                 diag = fm10k_dev_rx_queue_start(dev, i);
1127                 if (diag != 0) {
1128                         int j;
1129                         for (j = 0; j < i; ++j)
1130                                 rx_queue_clean(dev->data->rx_queues[j]);
1131                         return diag;
1132                 }
1133         }
1134
1135         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1136                 struct fm10k_tx_queue *txq;
1137                 txq = dev->data->tx_queues[i];
1138
1139                 if (txq->tx_deferred_start)
1140                         continue;
1141                 diag = fm10k_dev_tx_queue_start(dev, i);
1142                 if (diag != 0) {
1143                         int j;
1144                         for (j = 0; j < i; ++j)
1145                                 tx_queue_clean(dev->data->tx_queues[j]);
1146                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1147                                 rx_queue_clean(dev->data->rx_queues[j]);
1148                         return diag;
1149                 }
1150         }
1151
1152         /* Update default vlan when not in VMDQ mode */
1153         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1154                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1155
1156         fm10k_link_update(dev, 0);
1157
1158         return 0;
1159 }
1160
1161 static void
1162 fm10k_dev_stop(struct rte_eth_dev *dev)
1163 {
1164         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1165         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1166         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1167         int i;
1168
1169         PMD_INIT_FUNC_TRACE();
1170
1171         if (dev->data->tx_queues)
1172                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1173                         fm10k_dev_tx_queue_stop(dev, i);
1174
1175         if (dev->data->rx_queues)
1176                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1177                         fm10k_dev_rx_queue_stop(dev, i);
1178
1179         /* Disable datapath event */
1180         if (rte_intr_dp_is_en(intr_handle)) {
1181                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1182                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1183                                 3 << FM10K_RXINT_TIMER_SHIFT);
1184                         if (hw->mac.type == fm10k_mac_pf)
1185                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1186                                         FM10K_ITR_MASK_SET);
1187                         else
1188                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1189                                         FM10K_ITR_MASK_SET);
1190                 }
1191         }
1192         /* Clean datapath event and queue/vec mapping */
1193         rte_intr_efd_disable(intr_handle);
1194         rte_free(intr_handle->intr_vec);
1195         intr_handle->intr_vec = NULL;
1196 }
1197
1198 static void
1199 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1200 {
1201         int i;
1202
1203         PMD_INIT_FUNC_TRACE();
1204
1205         if (dev->data->tx_queues) {
1206                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1207                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1208
1209                         tx_queue_free(txq);
1210                 }
1211         }
1212
1213         if (dev->data->rx_queues) {
1214                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1215                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1216         }
1217 }
1218
1219 static void
1220 fm10k_dev_close(struct rte_eth_dev *dev)
1221 {
1222         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1223
1224         PMD_INIT_FUNC_TRACE();
1225
1226         fm10k_mbx_lock(hw);
1227         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1228                 MAX_LPORT_NUM, false);
1229         fm10k_mbx_unlock(hw);
1230
1231         /* allow 100ms for device to quiesce */
1232         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1233
1234         /* Stop mailbox service first */
1235         fm10k_close_mbx_service(hw);
1236         fm10k_dev_stop(dev);
1237         fm10k_dev_queue_release(dev);
1238         fm10k_stop_hw(hw);
1239 }
1240
1241 static int
1242 fm10k_link_update(struct rte_eth_dev *dev,
1243         __rte_unused int wait_to_complete)
1244 {
1245         struct fm10k_dev_info *dev_info =
1246                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1247         PMD_INIT_FUNC_TRACE();
1248
1249         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1250         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1251         dev->data->dev_link.link_status =
1252                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1253         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1254
1255         return 0;
1256 }
1257
1258 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1259         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1260 {
1261         unsigned i, q;
1262         unsigned count = 0;
1263
1264         if (xstats_names != NULL) {
1265                 /* Note: limit checked in rte_eth_xstats_names() */
1266
1267                 /* Global stats */
1268                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1269                         snprintf(xstats_names[count].name,
1270                                 sizeof(xstats_names[count].name),
1271                                 "%s", fm10k_hw_stats_strings[count].name);
1272                         count++;
1273                 }
1274
1275                 /* PF queue stats */
1276                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1277                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1278                                 snprintf(xstats_names[count].name,
1279                                         sizeof(xstats_names[count].name),
1280                                         "rx_q%u_%s", q,
1281                                         fm10k_hw_stats_rx_q_strings[i].name);
1282                                 count++;
1283                         }
1284                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1285                                 snprintf(xstats_names[count].name,
1286                                         sizeof(xstats_names[count].name),
1287                                         "tx_q%u_%s", q,
1288                                         fm10k_hw_stats_tx_q_strings[i].name);
1289                                 count++;
1290                         }
1291                 }
1292         }
1293         return FM10K_NB_XSTATS;
1294 }
1295
1296 static int
1297 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1298                  unsigned n)
1299 {
1300         struct fm10k_hw_stats *hw_stats =
1301                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1302         unsigned i, q, count = 0;
1303
1304         if (n < FM10K_NB_XSTATS)
1305                 return FM10K_NB_XSTATS;
1306
1307         /* Global stats */
1308         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1309                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1310                         fm10k_hw_stats_strings[count].offset);
1311                 xstats[count].id = count;
1312                 count++;
1313         }
1314
1315         /* PF queue stats */
1316         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1317                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1318                         xstats[count].value =
1319                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1320                                 fm10k_hw_stats_rx_q_strings[i].offset);
1321                         xstats[count].id = count;
1322                         count++;
1323                 }
1324                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1325                         xstats[count].value =
1326                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1327                                 fm10k_hw_stats_tx_q_strings[i].offset);
1328                         xstats[count].id = count;
1329                         count++;
1330                 }
1331         }
1332
1333         return FM10K_NB_XSTATS;
1334 }
1335
1336 static int
1337 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1338 {
1339         uint64_t ipackets, opackets, ibytes, obytes;
1340         struct fm10k_hw *hw =
1341                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1342         struct fm10k_hw_stats *hw_stats =
1343                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1344         int i;
1345
1346         PMD_INIT_FUNC_TRACE();
1347
1348         fm10k_update_hw_stats(hw, hw_stats);
1349
1350         ipackets = opackets = ibytes = obytes = 0;
1351         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1352                 (i < hw->mac.max_queues); ++i) {
1353                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1354                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1355                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1356                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1357                 ipackets += stats->q_ipackets[i];
1358                 opackets += stats->q_opackets[i];
1359                 ibytes   += stats->q_ibytes[i];
1360                 obytes   += stats->q_obytes[i];
1361         }
1362         stats->ipackets = ipackets;
1363         stats->opackets = opackets;
1364         stats->ibytes = ibytes;
1365         stats->obytes = obytes;
1366         return 0;
1367 }
1368
1369 static void
1370 fm10k_stats_reset(struct rte_eth_dev *dev)
1371 {
1372         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1373         struct fm10k_hw_stats *hw_stats =
1374                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1375
1376         PMD_INIT_FUNC_TRACE();
1377
1378         memset(hw_stats, 0, sizeof(*hw_stats));
1379         fm10k_rebind_hw_stats(hw, hw_stats);
1380 }
1381
1382 static void
1383 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1384         struct rte_eth_dev_info *dev_info)
1385 {
1386         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1387         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1388
1389         PMD_INIT_FUNC_TRACE();
1390
1391         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1392         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1393         dev_info->max_rx_queues      = hw->mac.max_queues;
1394         dev_info->max_tx_queues      = hw->mac.max_queues;
1395         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1396         dev_info->max_hash_mac_addrs = 0;
1397         dev_info->max_vfs            = pdev->max_vfs;
1398         dev_info->vmdq_pool_base     = 0;
1399         dev_info->vmdq_queue_base    = 0;
1400         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1401         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1402         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1403         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1404                                     dev_info->rx_queue_offload_capa;
1405         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1406         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1407                                     dev_info->tx_queue_offload_capa;
1408
1409         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1410         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1411
1412         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1413                 .rx_thresh = {
1414                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1415                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1416                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1417                 },
1418                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1419                 .rx_drop_en = 0,
1420                 .offloads = 0,
1421         };
1422
1423         dev_info->default_txconf = (struct rte_eth_txconf) {
1424                 .tx_thresh = {
1425                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1426                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1427                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1428                 },
1429                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1430                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1431                 .offloads = 0,
1432         };
1433
1434         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1435                 .nb_max = FM10K_MAX_RX_DESC,
1436                 .nb_min = FM10K_MIN_RX_DESC,
1437                 .nb_align = FM10K_MULT_RX_DESC,
1438         };
1439
1440         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1441                 .nb_max = FM10K_MAX_TX_DESC,
1442                 .nb_min = FM10K_MIN_TX_DESC,
1443                 .nb_align = FM10K_MULT_TX_DESC,
1444                 .nb_seg_max = FM10K_TX_MAX_SEG,
1445                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1446         };
1447
1448         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1449                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1450                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1451 }
1452
1453 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1454 static const uint32_t *
1455 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1456 {
1457         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1458             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1459                 static uint32_t ptypes[] = {
1460                         /* refers to rx_desc_to_ol_flags() */
1461                         RTE_PTYPE_L2_ETHER,
1462                         RTE_PTYPE_L3_IPV4,
1463                         RTE_PTYPE_L3_IPV4_EXT,
1464                         RTE_PTYPE_L3_IPV6,
1465                         RTE_PTYPE_L3_IPV6_EXT,
1466                         RTE_PTYPE_L4_TCP,
1467                         RTE_PTYPE_L4_UDP,
1468                         RTE_PTYPE_UNKNOWN
1469                 };
1470
1471                 return ptypes;
1472         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1473                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1474                 static uint32_t ptypes_vec[] = {
1475                         /* refers to fm10k_desc_to_pktype_v() */
1476                         RTE_PTYPE_L3_IPV4,
1477                         RTE_PTYPE_L3_IPV4_EXT,
1478                         RTE_PTYPE_L3_IPV6,
1479                         RTE_PTYPE_L3_IPV6_EXT,
1480                         RTE_PTYPE_L4_TCP,
1481                         RTE_PTYPE_L4_UDP,
1482                         RTE_PTYPE_TUNNEL_GENEVE,
1483                         RTE_PTYPE_TUNNEL_NVGRE,
1484                         RTE_PTYPE_TUNNEL_VXLAN,
1485                         RTE_PTYPE_TUNNEL_GRE,
1486                         RTE_PTYPE_UNKNOWN
1487                 };
1488
1489                 return ptypes_vec;
1490         }
1491
1492         return NULL;
1493 }
1494 #else
1495 static const uint32_t *
1496 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1497 {
1498         return NULL;
1499 }
1500 #endif
1501
1502 static int
1503 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1504 {
1505         s32 result;
1506         uint16_t mac_num = 0;
1507         uint32_t vid_idx, vid_bit, mac_index;
1508         struct fm10k_hw *hw;
1509         struct fm10k_macvlan_filter_info *macvlan;
1510         struct rte_eth_dev_data *data = dev->data;
1511
1512         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1513         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1514
1515         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1516                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1517                 return -EINVAL;
1518         }
1519
1520         if (vlan_id > ETH_VLAN_ID_MAX) {
1521                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1522                 return -EINVAL;
1523         }
1524
1525         vid_idx = FM10K_VFTA_IDX(vlan_id);
1526         vid_bit = FM10K_VFTA_BIT(vlan_id);
1527         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1528         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1529                 return 0;
1530         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1531         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1532                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1533                         "in the VLAN filter table");
1534                 return -EINVAL;
1535         }
1536
1537         fm10k_mbx_lock(hw);
1538         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1539         fm10k_mbx_unlock(hw);
1540         if (result != FM10K_SUCCESS) {
1541                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1542                 return -EIO;
1543         }
1544
1545         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1546                         (result == FM10K_SUCCESS); mac_index++) {
1547                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1548                         continue;
1549                 if (mac_num > macvlan->mac_num - 1) {
1550                         PMD_INIT_LOG(ERR, "MAC address number "
1551                                         "not match");
1552                         break;
1553                 }
1554                 fm10k_mbx_lock(hw);
1555                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1556                         data->mac_addrs[mac_index].addr_bytes,
1557                         vlan_id, on, 0);
1558                 fm10k_mbx_unlock(hw);
1559                 mac_num++;
1560         }
1561         if (result != FM10K_SUCCESS) {
1562                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1563                 return -EIO;
1564         }
1565
1566         if (on) {
1567                 macvlan->vlan_num++;
1568                 macvlan->vfta[vid_idx] |= vid_bit;
1569         } else {
1570                 macvlan->vlan_num--;
1571                 macvlan->vfta[vid_idx] &= ~vid_bit;
1572         }
1573         return 0;
1574 }
1575
1576 static int
1577 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1578 {
1579         if (mask & ETH_VLAN_STRIP_MASK) {
1580                 if (!(dev->data->dev_conf.rxmode.offloads &
1581                         DEV_RX_OFFLOAD_VLAN_STRIP))
1582                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1583                                         "always on in fm10k");
1584         }
1585
1586         if (mask & ETH_VLAN_EXTEND_MASK) {
1587                 if (dev->data->dev_conf.rxmode.offloads &
1588                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1589                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1590                                         "supported in fm10k");
1591         }
1592
1593         if (mask & ETH_VLAN_FILTER_MASK) {
1594                 if (!(dev->data->dev_conf.rxmode.offloads &
1595                         DEV_RX_OFFLOAD_VLAN_FILTER))
1596                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1597         }
1598
1599         return 0;
1600 }
1601
1602 /* Add/Remove a MAC address, and update filters to main VSI */
1603 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1604                 const u8 *mac, bool add, uint32_t pool)
1605 {
1606         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1607         struct fm10k_macvlan_filter_info *macvlan;
1608         uint32_t i, j, k;
1609
1610         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1611
1612         if (pool != MAIN_VSI_POOL_NUMBER) {
1613                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1614                         "mac to pool %u", pool);
1615                 return;
1616         }
1617         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1618                 if (!macvlan->vfta[j])
1619                         continue;
1620                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1621                         if (!(macvlan->vfta[j] & (1 << k)))
1622                                 continue;
1623                         if (i + 1 > macvlan->vlan_num) {
1624                                 PMD_INIT_LOG(ERR, "vlan number not match");
1625                                 return;
1626                         }
1627                         fm10k_mbx_lock(hw);
1628                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1629                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1630                         fm10k_mbx_unlock(hw);
1631                         i++;
1632                 }
1633         }
1634 }
1635
1636 /* Add/Remove a MAC address, and update filters to VMDQ */
1637 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1638                 const u8 *mac, bool add, uint32_t pool)
1639 {
1640         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1641         struct fm10k_macvlan_filter_info *macvlan;
1642         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1643         uint32_t i;
1644
1645         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1646         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1647
1648         if (pool > macvlan->nb_queue_pools) {
1649                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1650                         " Max pool is %u",
1651                         pool, macvlan->nb_queue_pools);
1652                 return;
1653         }
1654         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1655                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1656                         continue;
1657                 fm10k_mbx_lock(hw);
1658                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1659                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1660                 fm10k_mbx_unlock(hw);
1661         }
1662 }
1663
1664 /* Add/Remove a MAC address, and update filters */
1665 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1666                 const u8 *mac, bool add, uint32_t pool)
1667 {
1668         struct fm10k_macvlan_filter_info *macvlan;
1669
1670         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1671
1672         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1673                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1674         else
1675                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1676
1677         if (add)
1678                 macvlan->mac_num++;
1679         else
1680                 macvlan->mac_num--;
1681 }
1682
1683 /* Add a MAC address, and update filters */
1684 static int
1685 fm10k_macaddr_add(struct rte_eth_dev *dev,
1686                 struct ether_addr *mac_addr,
1687                 uint32_t index,
1688                 uint32_t pool)
1689 {
1690         struct fm10k_macvlan_filter_info *macvlan;
1691
1692         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1693         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1694         macvlan->mac_vmdq_id[index] = pool;
1695         return 0;
1696 }
1697
1698 /* Remove a MAC address, and update filters */
1699 static void
1700 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1701 {
1702         struct rte_eth_dev_data *data = dev->data;
1703         struct fm10k_macvlan_filter_info *macvlan;
1704
1705         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1706         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1707                         FALSE, macvlan->mac_vmdq_id[index]);
1708         macvlan->mac_vmdq_id[index] = 0;
1709 }
1710
1711 static inline int
1712 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1713 {
1714         if ((request < min) || (request > max) || ((request % mult) != 0))
1715                 return -1;
1716         else
1717                 return 0;
1718 }
1719
1720
1721 static inline int
1722 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1723 {
1724         if ((request < min) || (request > max) || ((div % request) != 0))
1725                 return -1;
1726         else
1727                 return 0;
1728 }
1729
1730 static inline int
1731 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1732 {
1733         uint16_t rx_free_thresh;
1734
1735         if (conf->rx_free_thresh == 0)
1736                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1737         else
1738                 rx_free_thresh = conf->rx_free_thresh;
1739
1740         /* make sure the requested threshold satisfies the constraints */
1741         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1742                         FM10K_RX_FREE_THRESH_MAX(q),
1743                         FM10K_RX_FREE_THRESH_DIV(q),
1744                         rx_free_thresh)) {
1745                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1746                         "less than or equal to %u, "
1747                         "greater than or equal to %u, "
1748                         "and a divisor of %u",
1749                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1750                         FM10K_RX_FREE_THRESH_MIN(q),
1751                         FM10K_RX_FREE_THRESH_DIV(q));
1752                 return -EINVAL;
1753         }
1754
1755         q->alloc_thresh = rx_free_thresh;
1756         q->drop_en = conf->rx_drop_en;
1757         q->rx_deferred_start = conf->rx_deferred_start;
1758
1759         return 0;
1760 }
1761
1762 /*
1763  * Hardware requires specific alignment for Rx packet buffers. At
1764  * least one of the following two conditions must be satisfied.
1765  *  1. Address is 512B aligned
1766  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1767  *
1768  * As such, the driver may need to adjust the DMA address within the
1769  * buffer by up to 512B.
1770  *
1771  * return 1 if the element size is valid, otherwise return 0.
1772  */
1773 static int
1774 mempool_element_size_valid(struct rte_mempool *mp)
1775 {
1776         uint32_t min_size;
1777
1778         /* elt_size includes mbuf header and headroom */
1779         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1780                         RTE_PKTMBUF_HEADROOM;
1781
1782         /* account for up to 512B of alignment */
1783         min_size -= FM10K_RX_DATABUF_ALIGN;
1784
1785         /* sanity check for overflow */
1786         if (min_size > mp->elt_size)
1787                 return 0;
1788
1789         /* size is valid */
1790         return 1;
1791 }
1792
1793 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1794 {
1795         RTE_SET_USED(dev);
1796
1797         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1798 }
1799
1800 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1801 {
1802         RTE_SET_USED(dev);
1803
1804         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1805                            DEV_RX_OFFLOAD_VLAN_FILTER |
1806                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1807                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1808                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1809                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1810                            DEV_RX_OFFLOAD_CRC_STRIP   |
1811                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1812 }
1813
1814 static int
1815 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1816         uint16_t nb_desc, unsigned int socket_id,
1817         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1818 {
1819         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1820         struct fm10k_dev_info *dev_info =
1821                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1822         struct fm10k_rx_queue *q;
1823         const struct rte_memzone *mz;
1824         uint64_t offloads;
1825
1826         PMD_INIT_FUNC_TRACE();
1827
1828         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1829
1830         /* make sure the mempool element size can account for alignment. */
1831         if (!mempool_element_size_valid(mp)) {
1832                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1833                 return -EINVAL;
1834         }
1835
1836         /* make sure a valid number of descriptors have been requested */
1837         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1838                                 FM10K_MULT_RX_DESC, nb_desc)) {
1839                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1840                         "less than or equal to %"PRIu32", "
1841                         "greater than or equal to %u, "
1842                         "and a multiple of %u",
1843                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1844                         FM10K_MULT_RX_DESC);
1845                 return -EINVAL;
1846         }
1847
1848         /*
1849          * if this queue existed already, free the associated memory. The
1850          * queue cannot be reused in case we need to allocate memory on
1851          * different socket than was previously used.
1852          */
1853         if (dev->data->rx_queues[queue_id] != NULL) {
1854                 rx_queue_free(dev->data->rx_queues[queue_id]);
1855                 dev->data->rx_queues[queue_id] = NULL;
1856         }
1857
1858         /* allocate memory for the queue structure */
1859         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1860                                 socket_id);
1861         if (q == NULL) {
1862                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1863                 return -ENOMEM;
1864         }
1865
1866         /* setup queue */
1867         q->mp = mp;
1868         q->nb_desc = nb_desc;
1869         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1870         q->port_id = dev->data->port_id;
1871         q->queue_id = queue_id;
1872         q->tail_ptr = (volatile uint32_t *)
1873                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1874         q->offloads = offloads;
1875         if (handle_rxconf(q, conf))
1876                 return -EINVAL;
1877
1878         /* allocate memory for the software ring */
1879         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1880                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1881                         RTE_CACHE_LINE_SIZE, socket_id);
1882         if (q->sw_ring == NULL) {
1883                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1884                 rte_free(q);
1885                 return -ENOMEM;
1886         }
1887
1888         /*
1889          * allocate memory for the hardware descriptor ring. A memzone large
1890          * enough to hold the maximum ring size is requested to allow for
1891          * resizing in later calls to the queue setup function.
1892          */
1893         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1894                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1895                                       socket_id);
1896         if (mz == NULL) {
1897                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1898                 rte_free(q->sw_ring);
1899                 rte_free(q);
1900                 return -ENOMEM;
1901         }
1902         q->hw_ring = mz->addr;
1903         q->hw_ring_phys_addr = mz->iova;
1904
1905         /* Check if number of descs satisfied Vector requirement */
1906         if (!rte_is_power_of_2(nb_desc)) {
1907                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1908                                     "preconditions - canceling the feature for "
1909                                     "the whole port[%d]",
1910                              q->queue_id, q->port_id);
1911                 dev_info->rx_vec_allowed = false;
1912         } else
1913                 fm10k_rxq_vec_setup(q);
1914
1915         dev->data->rx_queues[queue_id] = q;
1916         return 0;
1917 }
1918
1919 static void
1920 fm10k_rx_queue_release(void *queue)
1921 {
1922         PMD_INIT_FUNC_TRACE();
1923
1924         rx_queue_free(queue);
1925 }
1926
1927 static inline int
1928 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1929 {
1930         uint16_t tx_free_thresh;
1931         uint16_t tx_rs_thresh;
1932
1933         /* constraint MACROs require that tx_free_thresh is configured
1934          * before tx_rs_thresh */
1935         if (conf->tx_free_thresh == 0)
1936                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1937         else
1938                 tx_free_thresh = conf->tx_free_thresh;
1939
1940         /* make sure the requested threshold satisfies the constraints */
1941         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1942                         FM10K_TX_FREE_THRESH_MAX(q),
1943                         FM10K_TX_FREE_THRESH_DIV(q),
1944                         tx_free_thresh)) {
1945                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1946                         "less than or equal to %u, "
1947                         "greater than or equal to %u, "
1948                         "and a divisor of %u",
1949                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1950                         FM10K_TX_FREE_THRESH_MIN(q),
1951                         FM10K_TX_FREE_THRESH_DIV(q));
1952                 return -EINVAL;
1953         }
1954
1955         q->free_thresh = tx_free_thresh;
1956
1957         if (conf->tx_rs_thresh == 0)
1958                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1959         else
1960                 tx_rs_thresh = conf->tx_rs_thresh;
1961
1962         q->tx_deferred_start = conf->tx_deferred_start;
1963
1964         /* make sure the requested threshold satisfies the constraints */
1965         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1966                         FM10K_TX_RS_THRESH_MAX(q),
1967                         FM10K_TX_RS_THRESH_DIV(q),
1968                         tx_rs_thresh)) {
1969                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1970                         "less than or equal to %u, "
1971                         "greater than or equal to %u, "
1972                         "and a divisor of %u",
1973                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1974                         FM10K_TX_RS_THRESH_MIN(q),
1975                         FM10K_TX_RS_THRESH_DIV(q));
1976                 return -EINVAL;
1977         }
1978
1979         q->rs_thresh = tx_rs_thresh;
1980
1981         return 0;
1982 }
1983
1984 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1985 {
1986         RTE_SET_USED(dev);
1987
1988         return 0;
1989 }
1990
1991 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1992 {
1993         RTE_SET_USED(dev);
1994
1995         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1996                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1997                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1998                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1999                           DEV_TX_OFFLOAD_TCP_TSO);
2000 }
2001
2002 static int
2003 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2004         uint16_t nb_desc, unsigned int socket_id,
2005         const struct rte_eth_txconf *conf)
2006 {
2007         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2008         struct fm10k_tx_queue *q;
2009         const struct rte_memzone *mz;
2010         uint64_t offloads;
2011
2012         PMD_INIT_FUNC_TRACE();
2013
2014         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2015
2016         /* make sure a valid number of descriptors have been requested */
2017         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2018                                 FM10K_MULT_TX_DESC, nb_desc)) {
2019                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2020                         "less than or equal to %"PRIu32", "
2021                         "greater than or equal to %u, "
2022                         "and a multiple of %u",
2023                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2024                         FM10K_MULT_TX_DESC);
2025                 return -EINVAL;
2026         }
2027
2028         /*
2029          * if this queue existed already, free the associated memory. The
2030          * queue cannot be reused in case we need to allocate memory on
2031          * different socket than was previously used.
2032          */
2033         if (dev->data->tx_queues[queue_id] != NULL) {
2034                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2035
2036                 tx_queue_free(txq);
2037                 dev->data->tx_queues[queue_id] = NULL;
2038         }
2039
2040         /* allocate memory for the queue structure */
2041         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2042                                 socket_id);
2043         if (q == NULL) {
2044                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2045                 return -ENOMEM;
2046         }
2047
2048         /* setup queue */
2049         q->nb_desc = nb_desc;
2050         q->port_id = dev->data->port_id;
2051         q->queue_id = queue_id;
2052         q->offloads = offloads;
2053         q->ops = &def_txq_ops;
2054         q->tail_ptr = (volatile uint32_t *)
2055                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2056         if (handle_txconf(q, conf))
2057                 return -EINVAL;
2058
2059         /* allocate memory for the software ring */
2060         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2061                                         nb_desc * sizeof(struct rte_mbuf *),
2062                                         RTE_CACHE_LINE_SIZE, socket_id);
2063         if (q->sw_ring == NULL) {
2064                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2065                 rte_free(q);
2066                 return -ENOMEM;
2067         }
2068
2069         /*
2070          * allocate memory for the hardware descriptor ring. A memzone large
2071          * enough to hold the maximum ring size is requested to allow for
2072          * resizing in later calls to the queue setup function.
2073          */
2074         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2075                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2076                                       socket_id);
2077         if (mz == NULL) {
2078                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2079                 rte_free(q->sw_ring);
2080                 rte_free(q);
2081                 return -ENOMEM;
2082         }
2083         q->hw_ring = mz->addr;
2084         q->hw_ring_phys_addr = mz->iova;
2085
2086         /*
2087          * allocate memory for the RS bit tracker. Enough slots to hold the
2088          * descriptor index for each RS bit needing to be set are required.
2089          */
2090         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2091                                 ((nb_desc + 1) / q->rs_thresh) *
2092                                 sizeof(uint16_t),
2093                                 RTE_CACHE_LINE_SIZE, socket_id);
2094         if (q->rs_tracker.list == NULL) {
2095                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2096                 rte_free(q->sw_ring);
2097                 rte_free(q);
2098                 return -ENOMEM;
2099         }
2100
2101         dev->data->tx_queues[queue_id] = q;
2102         return 0;
2103 }
2104
2105 static void
2106 fm10k_tx_queue_release(void *queue)
2107 {
2108         struct fm10k_tx_queue *q = queue;
2109         PMD_INIT_FUNC_TRACE();
2110
2111         tx_queue_free(q);
2112 }
2113
2114 static int
2115 fm10k_reta_update(struct rte_eth_dev *dev,
2116                         struct rte_eth_rss_reta_entry64 *reta_conf,
2117                         uint16_t reta_size)
2118 {
2119         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2120         uint16_t i, j, idx, shift;
2121         uint8_t mask;
2122         uint32_t reta;
2123
2124         PMD_INIT_FUNC_TRACE();
2125
2126         if (reta_size > FM10K_MAX_RSS_INDICES) {
2127                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2128                         "(%d) doesn't match the number hardware can supported "
2129                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2130                 return -EINVAL;
2131         }
2132
2133         /*
2134          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2135          * 128-entries in 32 registers
2136          */
2137         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2138                 idx = i / RTE_RETA_GROUP_SIZE;
2139                 shift = i % RTE_RETA_GROUP_SIZE;
2140                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2141                                 BIT_MASK_PER_UINT32);
2142                 if (mask == 0)
2143                         continue;
2144
2145                 reta = 0;
2146                 if (mask != BIT_MASK_PER_UINT32)
2147                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2148
2149                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2150                         if (mask & (0x1 << j)) {
2151                                 if (mask != 0xF)
2152                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2153                                 reta |= reta_conf[idx].reta[shift + j] <<
2154                                                 (CHAR_BIT * j);
2155                         }
2156                 }
2157                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2158         }
2159
2160         return 0;
2161 }
2162
2163 static int
2164 fm10k_reta_query(struct rte_eth_dev *dev,
2165                         struct rte_eth_rss_reta_entry64 *reta_conf,
2166                         uint16_t reta_size)
2167 {
2168         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2169         uint16_t i, j, idx, shift;
2170         uint8_t mask;
2171         uint32_t reta;
2172
2173         PMD_INIT_FUNC_TRACE();
2174
2175         if (reta_size < FM10K_MAX_RSS_INDICES) {
2176                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2177                         "(%d) doesn't match the number hardware can supported "
2178                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2179                 return -EINVAL;
2180         }
2181
2182         /*
2183          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2184          * 128-entries in 32 registers
2185          */
2186         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2187                 idx = i / RTE_RETA_GROUP_SIZE;
2188                 shift = i % RTE_RETA_GROUP_SIZE;
2189                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2190                                 BIT_MASK_PER_UINT32);
2191                 if (mask == 0)
2192                         continue;
2193
2194                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2195                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2196                         if (mask & (0x1 << j))
2197                                 reta_conf[idx].reta[shift + j] = ((reta >>
2198                                         CHAR_BIT * j) & UINT8_MAX);
2199                 }
2200         }
2201
2202         return 0;
2203 }
2204
2205 static int
2206 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2207         struct rte_eth_rss_conf *rss_conf)
2208 {
2209         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2210         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2211         uint32_t mrqc;
2212         uint64_t hf = rss_conf->rss_hf;
2213         int i;
2214
2215         PMD_INIT_FUNC_TRACE();
2216
2217         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2218                                 FM10K_RSSRK_ENTRIES_PER_REG))
2219                 return -EINVAL;
2220
2221         if (hf == 0)
2222                 return -EINVAL;
2223
2224         mrqc = 0;
2225         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2226         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2227         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2228         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2229         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2230         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2231         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2232         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2233         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2234
2235         /* If the mapping doesn't fit any supported, return */
2236         if (mrqc == 0)
2237                 return -EINVAL;
2238
2239         if (key != NULL)
2240                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2241                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2242
2243         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2244
2245         return 0;
2246 }
2247
2248 static int
2249 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2250         struct rte_eth_rss_conf *rss_conf)
2251 {
2252         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2253         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2254         uint32_t mrqc;
2255         uint64_t hf;
2256         int i;
2257
2258         PMD_INIT_FUNC_TRACE();
2259
2260         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2261                                 FM10K_RSSRK_ENTRIES_PER_REG))
2262                 return -EINVAL;
2263
2264         if (key != NULL)
2265                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2266                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2267
2268         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2269         hf = 0;
2270         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2271         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2272         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2273         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2274         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2275         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2276         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2277         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2278         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2279
2280         rss_conf->rss_hf = hf;
2281
2282         return 0;
2283 }
2284
2285 static void
2286 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2287 {
2288         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2289         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2290
2291         /* Bind all local non-queue interrupt to vector 0 */
2292         int_map |= FM10K_MISC_VEC_ID;
2293
2294         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2298         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2299         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2300
2301         /* Enable misc causes */
2302         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2303                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2304                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2305                                 FM10K_EIMR_ENABLE(MAILBOX) |
2306                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2307                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2308                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2309                                 FM10K_EIMR_ENABLE(VFLR));
2310
2311         /* Enable ITR 0 */
2312         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2313                                         FM10K_ITR_MASK_CLEAR);
2314         FM10K_WRITE_FLUSH(hw);
2315 }
2316
2317 static void
2318 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2319 {
2320         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2321         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2322
2323         int_map |= FM10K_MISC_VEC_ID;
2324
2325         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2326         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2327         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2328         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2329         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2330         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2331
2332         /* Disable misc causes */
2333         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2334                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2335                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2336                                 FM10K_EIMR_DISABLE(MAILBOX) |
2337                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2338                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2339                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2340                                 FM10K_EIMR_DISABLE(VFLR));
2341
2342         /* Disable ITR 0 */
2343         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2344         FM10K_WRITE_FLUSH(hw);
2345 }
2346
2347 static void
2348 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2349 {
2350         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2351         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2352
2353         /* Bind all local non-queue interrupt to vector 0 */
2354         int_map |= FM10K_MISC_VEC_ID;
2355
2356         /* Only INT 0 available, other 15 are reserved. */
2357         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2358
2359         /* Enable ITR 0 */
2360         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2361                                         FM10K_ITR_MASK_CLEAR);
2362         FM10K_WRITE_FLUSH(hw);
2363 }
2364
2365 static void
2366 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2367 {
2368         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2369         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2370
2371         int_map |= FM10K_MISC_VEC_ID;
2372
2373         /* Only INT 0 available, other 15 are reserved. */
2374         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2375
2376         /* Disable ITR 0 */
2377         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2378         FM10K_WRITE_FLUSH(hw);
2379 }
2380
2381 static int
2382 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2383 {
2384         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2385         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2386
2387         /* Enable ITR */
2388         if (hw->mac.type == fm10k_mac_pf)
2389                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2390                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2391         else
2392                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2393                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2394         rte_intr_enable(&pdev->intr_handle);
2395         return 0;
2396 }
2397
2398 static int
2399 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2400 {
2401         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2402         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2403
2404         /* Disable ITR */
2405         if (hw->mac.type == fm10k_mac_pf)
2406                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2407                         FM10K_ITR_MASK_SET);
2408         else
2409                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2410                         FM10K_ITR_MASK_SET);
2411         return 0;
2412 }
2413
2414 static int
2415 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2416 {
2417         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2418         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2419         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2420         uint32_t intr_vector, vec;
2421         uint16_t queue_id;
2422         int result = 0;
2423
2424         /* fm10k needs one separate interrupt for mailbox,
2425          * so only drivers which support multiple interrupt vectors
2426          * e.g. vfio-pci can work for fm10k interrupt mode
2427          */
2428         if (!rte_intr_cap_multiple(intr_handle) ||
2429                         dev->data->dev_conf.intr_conf.rxq == 0)
2430                 return result;
2431
2432         intr_vector = dev->data->nb_rx_queues;
2433
2434         /* disable interrupt first */
2435         rte_intr_disable(intr_handle);
2436         if (hw->mac.type == fm10k_mac_pf)
2437                 fm10k_dev_disable_intr_pf(dev);
2438         else
2439                 fm10k_dev_disable_intr_vf(dev);
2440
2441         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2442                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2443                 result = -EIO;
2444         }
2445
2446         if (rte_intr_dp_is_en(intr_handle) && !result) {
2447                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2448                         dev->data->nb_rx_queues * sizeof(int), 0);
2449                 if (intr_handle->intr_vec) {
2450                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2451                                         queue_id < dev->data->nb_rx_queues;
2452                                         queue_id++) {
2453                                 intr_handle->intr_vec[queue_id] = vec;
2454                                 if (vec < intr_handle->nb_efd - 1
2455                                                 + FM10K_RX_VEC_START)
2456                                         vec++;
2457                         }
2458                 } else {
2459                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2460                                 " intr_vec", dev->data->nb_rx_queues);
2461                         rte_intr_efd_disable(intr_handle);
2462                         result = -ENOMEM;
2463                 }
2464         }
2465
2466         if (hw->mac.type == fm10k_mac_pf)
2467                 fm10k_dev_enable_intr_pf(dev);
2468         else
2469                 fm10k_dev_enable_intr_vf(dev);
2470         rte_intr_enable(intr_handle);
2471         hw->mac.ops.update_int_moderator(hw);
2472         return result;
2473 }
2474
2475 static int
2476 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2477 {
2478         struct fm10k_fault fault;
2479         int err;
2480         const char *estr = "Unknown error";
2481
2482         /* Process PCA fault */
2483         if (eicr & FM10K_EICR_PCA_FAULT) {
2484                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2485                 if (err)
2486                         goto error;
2487                 switch (fault.type) {
2488                 case PCA_NO_FAULT:
2489                         estr = "PCA_NO_FAULT"; break;
2490                 case PCA_UNMAPPED_ADDR:
2491                         estr = "PCA_UNMAPPED_ADDR"; break;
2492                 case PCA_BAD_QACCESS_PF:
2493                         estr = "PCA_BAD_QACCESS_PF"; break;
2494                 case PCA_BAD_QACCESS_VF:
2495                         estr = "PCA_BAD_QACCESS_VF"; break;
2496                 case PCA_MALICIOUS_REQ:
2497                         estr = "PCA_MALICIOUS_REQ"; break;
2498                 case PCA_POISONED_TLP:
2499                         estr = "PCA_POISONED_TLP"; break;
2500                 case PCA_TLP_ABORT:
2501                         estr = "PCA_TLP_ABORT"; break;
2502                 default:
2503                         goto error;
2504                 }
2505                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2506                         estr, fault.func ? "VF" : "PF", fault.func,
2507                         fault.address, fault.specinfo);
2508         }
2509
2510         /* Process THI fault */
2511         if (eicr & FM10K_EICR_THI_FAULT) {
2512                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2513                 if (err)
2514                         goto error;
2515                 switch (fault.type) {
2516                 case THI_NO_FAULT:
2517                         estr = "THI_NO_FAULT"; break;
2518                 case THI_MAL_DIS_Q_FAULT:
2519                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2520                 default:
2521                         goto error;
2522                 }
2523                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2524                         estr, fault.func ? "VF" : "PF", fault.func,
2525                         fault.address, fault.specinfo);
2526         }
2527
2528         /* Process FUM fault */
2529         if (eicr & FM10K_EICR_FUM_FAULT) {
2530                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2531                 if (err)
2532                         goto error;
2533                 switch (fault.type) {
2534                 case FUM_NO_FAULT:
2535                         estr = "FUM_NO_FAULT"; break;
2536                 case FUM_UNMAPPED_ADDR:
2537                         estr = "FUM_UNMAPPED_ADDR"; break;
2538                 case FUM_POISONED_TLP:
2539                         estr = "FUM_POISONED_TLP"; break;
2540                 case FUM_BAD_VF_QACCESS:
2541                         estr = "FUM_BAD_VF_QACCESS"; break;
2542                 case FUM_ADD_DECODE_ERR:
2543                         estr = "FUM_ADD_DECODE_ERR"; break;
2544                 case FUM_RO_ERROR:
2545                         estr = "FUM_RO_ERROR"; break;
2546                 case FUM_QPRC_CRC_ERROR:
2547                         estr = "FUM_QPRC_CRC_ERROR"; break;
2548                 case FUM_CSR_TIMEOUT:
2549                         estr = "FUM_CSR_TIMEOUT"; break;
2550                 case FUM_INVALID_TYPE:
2551                         estr = "FUM_INVALID_TYPE"; break;
2552                 case FUM_INVALID_LENGTH:
2553                         estr = "FUM_INVALID_LENGTH"; break;
2554                 case FUM_INVALID_BE:
2555                         estr = "FUM_INVALID_BE"; break;
2556                 case FUM_INVALID_ALIGN:
2557                         estr = "FUM_INVALID_ALIGN"; break;
2558                 default:
2559                         goto error;
2560                 }
2561                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2562                         estr, fault.func ? "VF" : "PF", fault.func,
2563                         fault.address, fault.specinfo);
2564         }
2565
2566         return 0;
2567 error:
2568         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2569         return err;
2570 }
2571
2572 /**
2573  * PF interrupt handler triggered by NIC for handling specific interrupt.
2574  *
2575  * @param handle
2576  *  Pointer to interrupt handle.
2577  * @param param
2578  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2579  *
2580  * @return
2581  *  void
2582  */
2583 static void
2584 fm10k_dev_interrupt_handler_pf(void *param)
2585 {
2586         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2587         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2588         uint32_t cause, status;
2589         struct fm10k_dev_info *dev_info =
2590                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2591         int status_mbx;
2592         s32 err;
2593
2594         if (hw->mac.type != fm10k_mac_pf)
2595                 return;
2596
2597         cause = FM10K_READ_REG(hw, FM10K_EICR);
2598
2599         /* Handle PCI fault cases */
2600         if (cause & FM10K_EICR_FAULT_MASK) {
2601                 PMD_INIT_LOG(ERR, "INT: find fault!");
2602                 fm10k_dev_handle_fault(hw, cause);
2603         }
2604
2605         /* Handle switch up/down */
2606         if (cause & FM10K_EICR_SWITCHNOTREADY)
2607                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2608
2609         if (cause & FM10K_EICR_SWITCHREADY) {
2610                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2611                 if (dev_info->sm_down == 1) {
2612                         fm10k_mbx_lock(hw);
2613
2614                         /* For recreating logical ports */
2615                         status_mbx = hw->mac.ops.update_lport_state(hw,
2616                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2617                         if (status_mbx == FM10K_SUCCESS)
2618                                 PMD_INIT_LOG(INFO,
2619                                         "INT: Recreated Logical port");
2620                         else
2621                                 PMD_INIT_LOG(INFO,
2622                                         "INT: Logical ports weren't recreated");
2623
2624                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2625                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2626                         if (status_mbx != FM10K_SUCCESS)
2627                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2628
2629                         fm10k_mbx_unlock(hw);
2630
2631                         /* first clear the internal SW recording structure */
2632                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2633                                                 ETH_MQ_RX_VMDQ_FLAG))
2634                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2635                                         false);
2636
2637                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2638                                         MAIN_VSI_POOL_NUMBER);
2639
2640                         /*
2641                          * Add default mac address and vlan for the logical
2642                          * ports that have been created, leave to the
2643                          * application to fully recover Rx filtering.
2644                          */
2645                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2646                                         MAIN_VSI_POOL_NUMBER);
2647
2648                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2649                                                 ETH_MQ_RX_VMDQ_FLAG))
2650                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2651                                         true);
2652
2653                         dev_info->sm_down = 0;
2654                         _rte_eth_dev_callback_process(dev,
2655                                         RTE_ETH_EVENT_INTR_LSC,
2656                                         NULL);
2657                 }
2658         }
2659
2660         /* Handle mailbox message */
2661         fm10k_mbx_lock(hw);
2662         err = hw->mbx.ops.process(hw, &hw->mbx);
2663         fm10k_mbx_unlock(hw);
2664
2665         if (err == FM10K_ERR_RESET_REQUESTED) {
2666                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2667                 dev_info->sm_down = 1;
2668                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2669                                 NULL);
2670         }
2671
2672         /* Handle SRAM error */
2673         if (cause & FM10K_EICR_SRAMERROR) {
2674                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2675
2676                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2677                 /* Write to clear pending bits */
2678                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2679
2680                 /* Todo: print out error message after shared code  updates */
2681         }
2682
2683         /* Clear these 3 events if having any */
2684         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2685                  FM10K_EICR_SWITCHREADY;
2686         if (cause)
2687                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2688
2689         /* Re-enable interrupt from device side */
2690         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2691                                         FM10K_ITR_MASK_CLEAR);
2692         /* Re-enable interrupt from host side */
2693         rte_intr_enable(dev->intr_handle);
2694 }
2695
2696 /**
2697  * VF interrupt handler triggered by NIC for handling specific interrupt.
2698  *
2699  * @param handle
2700  *  Pointer to interrupt handle.
2701  * @param param
2702  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2703  *
2704  * @return
2705  *  void
2706  */
2707 static void
2708 fm10k_dev_interrupt_handler_vf(void *param)
2709 {
2710         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2711         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2712         struct fm10k_mbx_info *mbx = &hw->mbx;
2713         struct fm10k_dev_info *dev_info =
2714                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2715         const enum fm10k_mbx_state state = mbx->state;
2716         int status_mbx;
2717
2718         if (hw->mac.type != fm10k_mac_vf)
2719                 return;
2720
2721         /* Handle mailbox message if lock is acquired */
2722         fm10k_mbx_lock(hw);
2723         hw->mbx.ops.process(hw, &hw->mbx);
2724         fm10k_mbx_unlock(hw);
2725
2726         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2727                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2728
2729                 fm10k_mbx_lock(hw);
2730                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2731                                 MAX_LPORT_NUM, 1);
2732                 fm10k_mbx_unlock(hw);
2733
2734                 /* Setting reset flag */
2735                 dev_info->sm_down = 1;
2736                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2737                                 NULL);
2738         }
2739
2740         if (dev_info->sm_down == 1 &&
2741                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2742                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2743                 fm10k_mbx_lock(hw);
2744                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2745                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2746                 if (status_mbx != FM10K_SUCCESS)
2747                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2748                 fm10k_mbx_unlock(hw);
2749
2750                 /* first clear the internal SW recording structure */
2751                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2752                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2753                                 MAIN_VSI_POOL_NUMBER);
2754
2755                 /*
2756                  * Add default mac address and vlan for the logical ports that
2757                  * have been created, leave to the application to fully recover
2758                  * Rx filtering.
2759                  */
2760                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2761                                 MAIN_VSI_POOL_NUMBER);
2762                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2763
2764                 dev_info->sm_down = 0;
2765                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2766                                 NULL);
2767         }
2768
2769         /* Re-enable interrupt from device side */
2770         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2771                                         FM10K_ITR_MASK_CLEAR);
2772         /* Re-enable interrupt from host side */
2773         rte_intr_enable(dev->intr_handle);
2774 }
2775
2776 /* Mailbox message handler in VF */
2777 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2778         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2779         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2780         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2781         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2782 };
2783
2784 static int
2785 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2786 {
2787         int err = 0;
2788
2789         /* Initialize mailbox lock */
2790         fm10k_mbx_initlock(hw);
2791
2792         /* Replace default message handler with new ones */
2793         if (hw->mac.type == fm10k_mac_vf)
2794                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2795
2796         if (err) {
2797                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2798                                 err);
2799                 return err;
2800         }
2801         /* Connect to SM for PF device or PF for VF device */
2802         return hw->mbx.ops.connect(hw, &hw->mbx);
2803 }
2804
2805 static void
2806 fm10k_close_mbx_service(struct fm10k_hw *hw)
2807 {
2808         /* Disconnect from SM for PF device or PF for VF device */
2809         hw->mbx.ops.disconnect(hw, &hw->mbx);
2810 }
2811
2812 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2813         .dev_configure          = fm10k_dev_configure,
2814         .dev_start              = fm10k_dev_start,
2815         .dev_stop               = fm10k_dev_stop,
2816         .dev_close              = fm10k_dev_close,
2817         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2818         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2819         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2820         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2821         .stats_get              = fm10k_stats_get,
2822         .xstats_get             = fm10k_xstats_get,
2823         .xstats_get_names       = fm10k_xstats_get_names,
2824         .stats_reset            = fm10k_stats_reset,
2825         .xstats_reset           = fm10k_stats_reset,
2826         .link_update            = fm10k_link_update,
2827         .dev_infos_get          = fm10k_dev_infos_get,
2828         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2829         .vlan_filter_set        = fm10k_vlan_filter_set,
2830         .vlan_offload_set       = fm10k_vlan_offload_set,
2831         .mac_addr_add           = fm10k_macaddr_add,
2832         .mac_addr_remove        = fm10k_macaddr_remove,
2833         .rx_queue_start         = fm10k_dev_rx_queue_start,
2834         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2835         .tx_queue_start         = fm10k_dev_tx_queue_start,
2836         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2837         .rx_queue_setup         = fm10k_rx_queue_setup,
2838         .rx_queue_release       = fm10k_rx_queue_release,
2839         .tx_queue_setup         = fm10k_tx_queue_setup,
2840         .tx_queue_release       = fm10k_tx_queue_release,
2841         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2842         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2843         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2844         .reta_update            = fm10k_reta_update,
2845         .reta_query             = fm10k_reta_query,
2846         .rss_hash_update        = fm10k_rss_hash_update,
2847         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2848 };
2849
2850 static int ftag_check_handler(__rte_unused const char *key,
2851                 const char *value, __rte_unused void *opaque)
2852 {
2853         if (strcmp(value, "1"))
2854                 return -1;
2855
2856         return 0;
2857 }
2858
2859 static int
2860 fm10k_check_ftag(struct rte_devargs *devargs)
2861 {
2862         struct rte_kvargs *kvlist;
2863         const char *ftag_key = "enable_ftag";
2864
2865         if (devargs == NULL)
2866                 return 0;
2867
2868         kvlist = rte_kvargs_parse(devargs->args, NULL);
2869         if (kvlist == NULL)
2870                 return 0;
2871
2872         if (!rte_kvargs_count(kvlist, ftag_key)) {
2873                 rte_kvargs_free(kvlist);
2874                 return 0;
2875         }
2876         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2877         if (rte_kvargs_process(kvlist, ftag_key,
2878                                 ftag_check_handler, NULL) < 0) {
2879                 rte_kvargs_free(kvlist);
2880                 return 0;
2881         }
2882         rte_kvargs_free(kvlist);
2883
2884         return 1;
2885 }
2886
2887 static uint16_t
2888 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2889                     uint16_t nb_pkts)
2890 {
2891         uint16_t nb_tx = 0;
2892         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2893
2894         while (nb_pkts) {
2895                 uint16_t ret, num;
2896
2897                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2898                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2899                                                  num);
2900                 nb_tx += ret;
2901                 nb_pkts -= ret;
2902                 if (ret < num)
2903                         break;
2904         }
2905
2906         return nb_tx;
2907 }
2908
2909 static void __attribute__((cold))
2910 fm10k_set_tx_function(struct rte_eth_dev *dev)
2911 {
2912         struct fm10k_tx_queue *txq;
2913         int i;
2914         int use_sse = 1;
2915         uint16_t tx_ftag_en = 0;
2916
2917         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2918                 /* primary process has set the ftag flag and offloads */
2919                 txq = dev->data->tx_queues[0];
2920                 if (fm10k_tx_vec_condition_check(txq)) {
2921                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2922                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2923                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2924                 } else {
2925                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2926                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2927                         dev->tx_pkt_prepare = NULL;
2928                 }
2929                 return;
2930         }
2931
2932         if (fm10k_check_ftag(dev->device->devargs))
2933                 tx_ftag_en = 1;
2934
2935         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2936                 txq = dev->data->tx_queues[i];
2937                 txq->tx_ftag_en = tx_ftag_en;
2938                 /* Check if Vector Tx is satisfied */
2939                 if (fm10k_tx_vec_condition_check(txq))
2940                         use_sse = 0;
2941         }
2942
2943         if (use_sse) {
2944                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2945                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2946                         txq = dev->data->tx_queues[i];
2947                         fm10k_txq_vec_setup(txq);
2948                 }
2949                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2950                 dev->tx_pkt_prepare = NULL;
2951         } else {
2952                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2953                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2954                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2955         }
2956 }
2957
2958 static void __attribute__((cold))
2959 fm10k_set_rx_function(struct rte_eth_dev *dev)
2960 {
2961         struct fm10k_dev_info *dev_info =
2962                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2963         uint16_t i, rx_using_sse;
2964         uint16_t rx_ftag_en = 0;
2965
2966         if (fm10k_check_ftag(dev->device->devargs))
2967                 rx_ftag_en = 1;
2968
2969         /* In order to allow Vector Rx there are a few configuration
2970          * conditions to be met.
2971          */
2972         if (!fm10k_rx_vec_condition_check(dev) &&
2973                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2974                 if (dev->data->scattered_rx)
2975                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2976                 else
2977                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2978         } else if (dev->data->scattered_rx)
2979                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2980         else
2981                 dev->rx_pkt_burst = fm10k_recv_pkts;
2982
2983         rx_using_sse =
2984                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2985                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2986
2987         if (rx_using_sse)
2988                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2989         else
2990                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2991
2992         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2993                 return;
2994
2995         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2996                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2997
2998                 rxq->rx_using_sse = rx_using_sse;
2999                 rxq->rx_ftag_en = rx_ftag_en;
3000         }
3001 }
3002
3003 static void
3004 fm10k_params_init(struct rte_eth_dev *dev)
3005 {
3006         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3007         struct fm10k_dev_info *info =
3008                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3009
3010         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3011          * there is no way to get link status without reading BAR4.  Until this
3012          * works, assume we have maximum bandwidth.
3013          * @todo - fix bus info
3014          */
3015         hw->bus_caps.speed = fm10k_bus_speed_8000;
3016         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3017         hw->bus_caps.payload = fm10k_bus_payload_512;
3018         hw->bus.speed = fm10k_bus_speed_8000;
3019         hw->bus.width = fm10k_bus_width_pcie_x8;
3020         hw->bus.payload = fm10k_bus_payload_256;
3021
3022         info->rx_vec_allowed = true;
3023 }
3024
3025 static int
3026 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3027 {
3028         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3029         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3030         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3031         int diag, i;
3032         struct fm10k_macvlan_filter_info *macvlan;
3033
3034         PMD_INIT_FUNC_TRACE();
3035
3036         dev->dev_ops = &fm10k_eth_dev_ops;
3037         dev->rx_pkt_burst = &fm10k_recv_pkts;
3038         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3039         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3040
3041         /*
3042          * Primary process does the whole initialization, for secondary
3043          * processes, we just select the same Rx and Tx function as primary.
3044          */
3045         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3046                 fm10k_set_rx_function(dev);
3047                 fm10k_set_tx_function(dev);
3048                 return 0;
3049         }
3050
3051         rte_eth_copy_pci_info(dev, pdev);
3052
3053         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3054         memset(macvlan, 0, sizeof(*macvlan));
3055         /* Vendor and Device ID need to be set before init of shared code */
3056         memset(hw, 0, sizeof(*hw));
3057         hw->device_id = pdev->id.device_id;
3058         hw->vendor_id = pdev->id.vendor_id;
3059         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3060         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3061         hw->revision_id = 0;
3062         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3063         if (hw->hw_addr == NULL) {
3064                 PMD_INIT_LOG(ERR, "Bad mem resource."
3065                         " Try to blacklist unused devices.");
3066                 return -EIO;
3067         }
3068
3069         /* Store fm10k_adapter pointer */
3070         hw->back = dev->data->dev_private;
3071
3072         /* Initialize the shared code */
3073         diag = fm10k_init_shared_code(hw);
3074         if (diag != FM10K_SUCCESS) {
3075                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3076                 return -EIO;
3077         }
3078
3079         /* Initialize parameters */
3080         fm10k_params_init(dev);
3081
3082         /* Initialize the hw */
3083         diag = fm10k_init_hw(hw);
3084         if (diag != FM10K_SUCCESS) {
3085                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3086                 return -EIO;
3087         }
3088
3089         /* Initialize MAC address(es) */
3090         dev->data->mac_addrs = rte_zmalloc("fm10k",
3091                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3092         if (dev->data->mac_addrs == NULL) {
3093                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3094                 return -ENOMEM;
3095         }
3096
3097         diag = fm10k_read_mac_addr(hw);
3098
3099         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3100                         &dev->data->mac_addrs[0]);
3101
3102         if (diag != FM10K_SUCCESS ||
3103                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3104
3105                 /* Generate a random addr */
3106                 eth_random_addr(hw->mac.addr);
3107                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3108                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3109                 &dev->data->mac_addrs[0]);
3110         }
3111
3112         /* Reset the hw statistics */
3113         fm10k_stats_reset(dev);
3114
3115         /* Reset the hw */
3116         diag = fm10k_reset_hw(hw);
3117         if (diag != FM10K_SUCCESS) {
3118                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3119                 return -EIO;
3120         }
3121
3122         /* Setup mailbox service */
3123         diag = fm10k_setup_mbx_service(hw);
3124         if (diag != FM10K_SUCCESS) {
3125                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3126                 return -EIO;
3127         }
3128
3129         /*PF/VF has different interrupt handling mechanism */
3130         if (hw->mac.type == fm10k_mac_pf) {
3131                 /* register callback func to eal lib */
3132                 rte_intr_callback_register(intr_handle,
3133                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3134
3135                 /* enable MISC interrupt */
3136                 fm10k_dev_enable_intr_pf(dev);
3137         } else { /* VF */
3138                 rte_intr_callback_register(intr_handle,
3139                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3140
3141                 fm10k_dev_enable_intr_vf(dev);
3142         }
3143
3144         /* Enable intr after callback registered */
3145         rte_intr_enable(intr_handle);
3146
3147         hw->mac.ops.update_int_moderator(hw);
3148
3149         /* Make sure Switch Manager is ready before going forward. */
3150         if (hw->mac.type == fm10k_mac_pf) {
3151                 int switch_ready = 0;
3152
3153                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3154                         fm10k_mbx_lock(hw);
3155                         hw->mac.ops.get_host_state(hw, &switch_ready);
3156                         fm10k_mbx_unlock(hw);
3157                         if (switch_ready)
3158                                 break;
3159                         /* Delay some time to acquire async LPORT_MAP info. */
3160                         rte_delay_us(WAIT_SWITCH_MSG_US);
3161                 }
3162
3163                 if (switch_ready == 0) {
3164                         PMD_INIT_LOG(ERR, "switch is not ready");
3165                         return -1;
3166                 }
3167         }
3168
3169         /*
3170          * Below function will trigger operations on mailbox, acquire lock to
3171          * avoid race condition from interrupt handler. Operations on mailbox
3172          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3173          * will handle and generate an interrupt to our side. Then,  FIFO in
3174          * mailbox will be touched.
3175          */
3176         fm10k_mbx_lock(hw);
3177         /* Enable port first */
3178         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3179                                         MAX_LPORT_NUM, 1);
3180
3181         /* Set unicast mode by default. App can change to other mode in other
3182          * API func.
3183          */
3184         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3185                                         FM10K_XCAST_MODE_NONE);
3186
3187         fm10k_mbx_unlock(hw);
3188
3189         /* Make sure default VID is ready before going forward. */
3190         if (hw->mac.type == fm10k_mac_pf) {
3191                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3192                         if (hw->mac.default_vid)
3193                                 break;
3194                         /* Delay some time to acquire async port VLAN info. */
3195                         rte_delay_us(WAIT_SWITCH_MSG_US);
3196                 }
3197
3198                 if (!hw->mac.default_vid) {
3199                         PMD_INIT_LOG(ERR, "default VID is not ready");
3200                         return -1;
3201                 }
3202         }
3203
3204         /* Add default mac address */
3205         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3206                 MAIN_VSI_POOL_NUMBER);
3207
3208         return 0;
3209 }
3210
3211 static int
3212 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3213 {
3214         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3215         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3216         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3217         PMD_INIT_FUNC_TRACE();
3218
3219         /* only uninitialize in the primary process */
3220         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3221                 return 0;
3222
3223         /* safe to close dev here */
3224         fm10k_dev_close(dev);
3225
3226         dev->dev_ops = NULL;
3227         dev->rx_pkt_burst = NULL;
3228         dev->tx_pkt_burst = NULL;
3229
3230         /* disable uio/vfio intr */
3231         rte_intr_disable(intr_handle);
3232
3233         /*PF/VF has different interrupt handling mechanism */
3234         if (hw->mac.type == fm10k_mac_pf) {
3235                 /* disable interrupt */
3236                 fm10k_dev_disable_intr_pf(dev);
3237
3238                 /* unregister callback func to eal lib */
3239                 rte_intr_callback_unregister(intr_handle,
3240                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3241         } else {
3242                 /* disable interrupt */
3243                 fm10k_dev_disable_intr_vf(dev);
3244
3245                 rte_intr_callback_unregister(intr_handle,
3246                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3247         }
3248
3249         /* free mac memory */
3250         if (dev->data->mac_addrs) {
3251                 rte_free(dev->data->mac_addrs);
3252                 dev->data->mac_addrs = NULL;
3253         }
3254
3255         memset(hw, 0, sizeof(*hw));
3256
3257         return 0;
3258 }
3259
3260 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3261         struct rte_pci_device *pci_dev)
3262 {
3263         return rte_eth_dev_pci_generic_probe(pci_dev,
3264                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3265 }
3266
3267 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3268 {
3269         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3270 }
3271
3272 /*
3273  * The set of PCI devices this driver supports. This driver will enable both PF
3274  * and SRIOV-VF devices.
3275  */
3276 static const struct rte_pci_id pci_id_fm10k_map[] = {
3277         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3278         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3279         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3280         { .vendor_id = 0, /* sentinel */ },
3281 };
3282
3283 static struct rte_pci_driver rte_pmd_fm10k = {
3284         .id_table = pci_id_fm10k_map,
3285         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3286                      RTE_PCI_DRV_IOVA_AS_VA,
3287         .probe = eth_fm10k_pci_probe,
3288         .remove = eth_fm10k_pci_remove,
3289 };
3290
3291 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3292 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3293 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3294
3295 RTE_INIT(fm10k_init_log);
3296 static void
3297 fm10k_init_log(void)
3298 {
3299         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3300         if (fm10k_logtype_init >= 0)
3301                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3302         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3303         if (fm10k_logtype_driver >= 0)
3304                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3305 }