drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 /* fls = find last set bit = 32 minus the number of leading zeros */
 468 #ifndef fls
 469 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 470 #endif
 471
 472 static void
 473 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 474 {
 475         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 476         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 477         uint32_t i;
 478
 479         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 480
 481         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 482                 if (!vmdq_conf->pool_map[i].pools)
 483                         continue;
 484                 fm10k_mbx_lock(hw);
 485                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 486                 fm10k_mbx_unlock(hw);
 487         }
 488 }
 489
 490 static void
 491 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 492 {
 493         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 494
 495         /* Add default mac address */
 496         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 497                 MAIN_VSI_POOL_NUMBER);
 498 }
 499
 500 static void
 501 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 502 {
 503         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 504         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 505         uint32_t mrqc, *key, i, reta, j;
 506         uint64_t hf;
 507
 508 #define RSS_KEY_SIZE 40
 509         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 510                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 511                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 512                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 513                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 514                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 515         };
 516
 517         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 518                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 519                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 520                 return;
 521         }
 522
 523         /* random key is rss_intel_key (default) or user provided (rss_key) */
 524         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 525                 key = (uint32_t *)rss_intel_key;
 526         else
 527                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 528
 529         /* Now fill our hash function seeds, 4 bytes at a time */
 530         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 531                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 532
 533         /*
 534          * Fill in redirection table
 535          * The byte-swap is needed because NIC registers are in
 536          * little-endian order.
 537          */
 538         reta = 0;
 539         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 540                 if (j == dev->data->nb_rx_queues)
 541                         j = 0;
 542                 reta = (reta << CHAR_BIT) | j;
 543                 if ((i & 3) == 3)
 544                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 545                                         rte_bswap32(reta));
 546         }
 547
 548         /*
 549          * Generate RSS hash based on packet types, TCP/UDP
 550          * port numbers and/or IPv4/v6 src and dst addresses
 551          */
 552         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 553         mrqc = 0;
 554         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 555         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 556         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 557         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 558         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 559         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 560         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 561         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 562         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 563
 564         if (mrqc == 0) {
 565                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 566                         "supported", hf);
 567                 return;
 568         }
 569
 570         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 571 }
 572
 573 static void
 574 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 575 {
 576         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 577         uint32_t i;
 578
 579         for (i = 0; i < nb_lport_new; i++) {
 580                 /* Set unicast mode by default. App can change
 581                  * to other mode in other API func.
 582                  */
 583                 fm10k_mbx_lock(hw);
 584                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 585                         FM10K_XCAST_MODE_NONE);
 586                 fm10k_mbx_unlock(hw);
 587         }
 588 }
 589
 590 static void
 591 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 592 {
 593         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 594         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 595         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 596         struct fm10k_macvlan_filter_info *macvlan;
 597         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 598         uint16_t nb_lport_new;
 599
 600         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 601         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 602
 603         fm10k_dev_rss_configure(dev);
 604
 605         /* only PF supports VMDQ */
 606         if (hw->mac.type != fm10k_mac_pf)
 607                 return;
 608
 609         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 610                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 611
 612         /* no pool number change, no need to update logic port and VLAN/MAC */
 613         if (macvlan->nb_queue_pools == nb_queue_pools)
 614                 return;
 615
 616         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 617         fm10k_dev_logic_port_update(dev, nb_lport_new);
 618
 619         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 620         memset(dev->data->mac_addrs, 0,
 621                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 622         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 623                 &dev->data->mac_addrs[0]);
 624         memset(macvlan, 0, sizeof(*macvlan));
 625         macvlan->nb_queue_pools = nb_queue_pools;
 626
 627         if (nb_queue_pools)
 628                 fm10k_dev_vmdq_rx_configure(dev);
 629         else
 630                 fm10k_dev_pf_main_vsi_reset(dev);
 631 }
 632
 633 static int
 634 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 635 {
 636         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 637         int i, ret;
 638         struct fm10k_tx_queue *txq;
 639         uint64_t base_addr;
 640         uint32_t size;
 641
 642         /* Disable TXINT to avoid possible interrupt */
 643         for (i = 0; i < hw->mac.max_queues; i++)
 644                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 645                                 3 << FM10K_TXINT_TIMER_SHIFT);
 646
 647         /* Setup TX queue */
 648         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 649                 txq = dev->data->tx_queues[i];
 650                 base_addr = txq->hw_ring_phys_addr;
 651                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 652
 653                 /* disable queue to avoid issues while updating state */
 654                 ret = tx_queue_disable(hw, i);
 655                 if (ret) {
 656                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 657                         return -1;
 658                 }
 659                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 660                  * register is read-only for VF.
 661                  */
 662                 if (fm10k_check_ftag(dev->device->devargs)) {
 663                         if (hw->mac.type == fm10k_mac_pf) {
 664                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 665                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 666                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 667                         } else {
 668                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 669                                 return -ENOTSUP;
 670                         }
 671                 }
 672
 673                 /* set location and size for descriptor ring */
 674                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 675                                 base_addr & UINT64_LOWER_32BITS_MASK);
 676                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 677                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 678                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 679
 680                 /* assign default SGLORT for each TX queue by PF */
 681                 if (hw->mac.type == fm10k_mac_pf)
 682                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 683         }
 684
 685         /* set up vector or scalar TX function as appropriate */
 686         fm10k_set_tx_function(dev);
 687
 688         return 0;
 689 }
 690
 691 static int
 692 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 693 {
 694         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 695         struct fm10k_macvlan_filter_info *macvlan;
 696         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 697         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 698         int i, ret;
 699         struct fm10k_rx_queue *rxq;
 700         uint64_t base_addr;
 701         uint32_t size;
 702         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 703         uint32_t logic_port = hw->mac.dglort_map;
 704         uint16_t buf_size;
 705         uint16_t queue_stride = 0;
 706
 707         /* enable RXINT for interrupt mode */
 708         i = 0;
 709         if (rte_intr_dp_is_en(intr_handle)) {
 710                 for (; i < dev->data->nb_rx_queues; i++) {
 711                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 712                         if (hw->mac.type == fm10k_mac_pf)
 713                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 714                                         FM10K_ITR_AUTOMASK |
 715                                         FM10K_ITR_MASK_CLEAR);
 716                         else
 717                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 718                                         FM10K_ITR_AUTOMASK |
 719                                         FM10K_ITR_MASK_CLEAR);
 720                 }
 721         }
 722         /* Disable other RXINT to avoid possible interrupt */
 723         for (; i < hw->mac.max_queues; i++)
 724                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 725                         3 << FM10K_RXINT_TIMER_SHIFT);
 726
 727         /* Setup RX queues */
 728         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 729                 rxq = dev->data->rx_queues[i];
 730                 base_addr = rxq->hw_ring_phys_addr;
 731                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 732
 733                 /* disable queue to avoid issues while updating state */
 734                 ret = rx_queue_disable(hw, i);
 735                 if (ret) {
 736                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 737                         return -1;
 738                 }
 739
 740                 /* Setup the Base and Length of the Rx Descriptor Ring */
 741                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 742                                 base_addr & UINT64_LOWER_32BITS_MASK);
 743                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 744                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 745                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 746
 747                 /* Configure the Rx buffer size for one buff without split */
 748                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 749                         RTE_PKTMBUF_HEADROOM);
 750                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 751                  * reserved for this purpose, and the worst case could be 511B.
 752                  * But SRR reg assumes all buffers have the same size. In order
 753                  * to fill the gap, we'll have to consider the worst case and
 754                  * assume 512B is reserved. If we don't do so, it's possible
 755                  * for HW to overwrite data to next mbuf.
 756                  */
 757                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 758
 759                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 760                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 761                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 762
 763                 /* It adds dual VLAN length for supporting dual VLAN */
 764                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 765                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 766                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 767                         uint32_t reg;
 768                         dev->data->scattered_rx = 1;
 769                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 770                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 771                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 772                 }
 773
 774                 /* Enable drop on empty, it's RO for VF */
 775                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 776                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 777
 778                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 779                 FM10K_WRITE_FLUSH(hw);
 780         }
 781
 782         /* Configure VMDQ/RSS if applicable */
 783         fm10k_dev_mq_rx_configure(dev);
 784
 785         /* Decide the best RX function */
 786         fm10k_set_rx_function(dev);
 787
 788         /* update RX_SGLORT for loopback suppress*/
 789         if (hw->mac.type != fm10k_mac_pf)
 790                 return 0;
 791         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 792         if (macvlan->nb_queue_pools)
 793                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 794         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 795                 if (i && queue_stride && !(i % queue_stride))
 796                         logic_port++;
 797                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 798         }
 799
 800         return 0;
 801 }
 802
 803 static int
 804 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 805 {
 806         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 807         int err;
 808         uint32_t reg;
 809         struct fm10k_rx_queue *rxq;
 810
 811         PMD_INIT_FUNC_TRACE();
 812
 813         rxq = dev->data->rx_queues[rx_queue_id];
 814         err = rx_queue_reset(rxq);
 815         if (err == -ENOMEM) {
 816                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 817                 return err;
 818         } else if (err == -EINVAL) {
 819                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 820                         " %d", err);
 821                 return err;
 822         }
 823
 824         /* Setup the HW Rx Head and Tail Descriptor Pointers
 825          * Note: this must be done AFTER the queue is enabled on real
 826          * hardware, but BEFORE the queue is enabled when using the
 827          * emulation platform. Do it in both places for now and remove
 828          * this comment and the following two register writes when the
 829          * emulation platform is no longer being used.
 830          */
 831         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 832         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 833
 834         /* Set PF ownership flag for PF devices */
 835         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 836         if (hw->mac.type == fm10k_mac_pf)
 837                 reg |= FM10K_RXQCTL_PF;
 838         reg |= FM10K_RXQCTL_ENABLE;
 839         /* enable RX queue */
 840         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 841         FM10K_WRITE_FLUSH(hw);
 842
 843         /* Setup the HW Rx Head and Tail Descriptor Pointers
 844          * Note: this must be done AFTER the queue is enabled
 845          */
 846         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 847         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 848         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 849
 850         return 0;
 851 }
 852
 853 static int
 854 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 855 {
 856         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 857
 858         PMD_INIT_FUNC_TRACE();
 859
 860         /* Disable RX queue */
 861         rx_queue_disable(hw, rx_queue_id);
 862
 863         /* Free mbuf and clean HW ring */
 864         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 865         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 866
 867         return 0;
 868 }
 869
 870 static int
 871 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 872 {
 873         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 874         /** @todo - this should be defined in the shared code */
 875 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 876         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 877         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 878
 879         PMD_INIT_FUNC_TRACE();
 880
 881         q->ops->reset(q);
 882
 883         /* reset head and tail pointers */
 884         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 885         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 886
 887         /* enable TX queue */
 888         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 889                                 FM10K_TXDCTL_ENABLE | txdctl);
 890         FM10K_WRITE_FLUSH(hw);
 891         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 892
 893         return 0;
 894 }
 895
 896 static int
 897 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 898 {
 899         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 900
 901         PMD_INIT_FUNC_TRACE();
 902
 903         tx_queue_disable(hw, tx_queue_id);
 904         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 905         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 906
 907         return 0;
 908 }
 909
 910 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 911 {
 912         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 913                 != FM10K_DGLORTMAP_NONE);
 914 }
 915
 916 static void
 917 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 918 {
 919         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 920         int status;
 921
 922         PMD_INIT_FUNC_TRACE();
 923
 924         /* Return if it didn't acquire valid glort range */
 925         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 926                 return;
 927
 928         fm10k_mbx_lock(hw);
 929         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 930                                 FM10K_XCAST_MODE_PROMISC);
 931         fm10k_mbx_unlock(hw);
 932
 933         if (status != FM10K_SUCCESS)
 934                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 935 }
 936
 937 static void
 938 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 939 {
 940         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 941         uint8_t mode;
 942         int status;
 943
 944         PMD_INIT_FUNC_TRACE();
 945
 946         /* Return if it didn't acquire valid glort range */
 947         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 948                 return;
 949
 950         if (dev->data->all_multicast == 1)
 951                 mode = FM10K_XCAST_MODE_ALLMULTI;
 952         else
 953                 mode = FM10K_XCAST_MODE_NONE;
 954
 955         fm10k_mbx_lock(hw);
 956         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 957                                 mode);
 958         fm10k_mbx_unlock(hw);
 959
 960         if (status != FM10K_SUCCESS)
 961                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 962 }
 963
 964 static void
 965 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 966 {
 967         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 968         int status;
 969
 970         PMD_INIT_FUNC_TRACE();
 971
 972         /* Return if it didn't acquire valid glort range */
 973         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 974                 return;
 975
 976         /* If promiscuous mode is enabled, it doesn't make sense to enable
 977          * allmulticast and disable promiscuous since fm10k only can select
 978          * one of the modes.
 979          */
 980         if (dev->data->promiscuous) {
 981                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 982                         "needn't enable allmulticast");
 983                 return;
 984         }
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 FM10K_XCAST_MODE_ALLMULTI);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         if (dev->data->promiscuous) {
1008                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1009                         "since promisc mode is enabled");
1010                 return;
1011         }
1012
1013         fm10k_mbx_lock(hw);
1014         /* Change mode to unicast mode */
1015         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1016                                 FM10K_XCAST_MODE_NONE);
1017         fm10k_mbx_unlock(hw);
1018
1019         if (status != FM10K_SUCCESS)
1020                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1021 }
1022
1023 static void
1024 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1025 {
1026         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1027         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1028         uint16_t nb_queue_pools;
1029         struct fm10k_macvlan_filter_info *macvlan;
1030
1031         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1032         nb_queue_pools = macvlan->nb_queue_pools;
1033         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1034         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1035
1036         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1037         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1038         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1039                         hw->mac.dglort_map;
1040         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1041         /* Configure VMDQ/RSS DGlort Decoder */
1042         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1043
1044         /* Flow Director configurations, only queue number is valid. */
1045         dglortdec = fls(dev->data->nb_rx_queues - 1);
1046         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1047                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1048         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1049         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1050
1051         /* Invalidate all other GLORT entries */
1052         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1053                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1054                                 FM10K_DGLORTMAP_NONE);
1055 }
1056
1057 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1058 static int
1059 fm10k_dev_start(struct rte_eth_dev *dev)
1060 {
1061         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1062         int i, diag;
1063
1064         PMD_INIT_FUNC_TRACE();
1065
1066         /* stop, init, then start the hw */
1067         diag = fm10k_stop_hw(hw);
1068         if (diag != FM10K_SUCCESS) {
1069                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1070                 return -EIO;
1071         }
1072
1073         diag = fm10k_init_hw(hw);
1074         if (diag != FM10K_SUCCESS) {
1075                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1076                 return -EIO;
1077         }
1078
1079         diag = fm10k_start_hw(hw);
1080         if (diag != FM10K_SUCCESS) {
1081                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1082                 return -EIO;
1083         }
1084
1085         diag = fm10k_dev_tx_init(dev);
1086         if (diag) {
1087                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1088                 return diag;
1089         }
1090
1091         if (fm10k_dev_rxq_interrupt_setup(dev))
1092                 return -EIO;
1093
1094         diag = fm10k_dev_rx_init(dev);
1095         if (diag) {
1096                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1097                 return diag;
1098         }
1099
1100         if (hw->mac.type == fm10k_mac_pf)
1101                 fm10k_dev_dglort_map_configure(dev);
1102
1103         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1104                 struct fm10k_rx_queue *rxq;
1105                 rxq = dev->data->rx_queues[i];
1106
1107                 if (rxq->rx_deferred_start)
1108                         continue;
1109                 diag = fm10k_dev_rx_queue_start(dev, i);
1110                 if (diag != 0) {
1111                         int j;
1112                         for (j = 0; j < i; ++j)
1113                                 rx_queue_clean(dev->data->rx_queues[j]);
1114                         return diag;
1115                 }
1116         }
1117
1118         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1119                 struct fm10k_tx_queue *txq;
1120                 txq = dev->data->tx_queues[i];
1121
1122                 if (txq->tx_deferred_start)
1123                         continue;
1124                 diag = fm10k_dev_tx_queue_start(dev, i);
1125                 if (diag != 0) {
1126                         int j;
1127                         for (j = 0; j < i; ++j)
1128                                 tx_queue_clean(dev->data->tx_queues[j]);
1129                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1130                                 rx_queue_clean(dev->data->rx_queues[j]);
1131                         return diag;
1132                 }
1133         }
1134
1135         /* Update default vlan when not in VMDQ mode */
1136         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1137                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1138
1139         fm10k_link_update(dev, 0);
1140
1141         return 0;
1142 }
1143
1144 static void
1145 fm10k_dev_stop(struct rte_eth_dev *dev)
1146 {
1147         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1148         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1149         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1150         int i;
1151
1152         PMD_INIT_FUNC_TRACE();
1153
1154         if (dev->data->tx_queues)
1155                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1156                         fm10k_dev_tx_queue_stop(dev, i);
1157
1158         if (dev->data->rx_queues)
1159                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1160                         fm10k_dev_rx_queue_stop(dev, i);
1161
1162         /* Disable datapath event */
1163         if (rte_intr_dp_is_en(intr_handle)) {
1164                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1165                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1166                                 3 << FM10K_RXINT_TIMER_SHIFT);
1167                         if (hw->mac.type == fm10k_mac_pf)
1168                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1169                                         FM10K_ITR_MASK_SET);
1170                         else
1171                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1172                                         FM10K_ITR_MASK_SET);
1173                 }
1174         }
1175         /* Clean datapath event and queue/vec mapping */
1176         rte_intr_efd_disable(intr_handle);
1177         rte_free(intr_handle->intr_vec);
1178         intr_handle->intr_vec = NULL;
1179 }
1180
1181 static void
1182 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1183 {
1184         int i;
1185
1186         PMD_INIT_FUNC_TRACE();
1187
1188         if (dev->data->tx_queues) {
1189                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1190                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1191
1192                         tx_queue_free(txq);
1193                 }
1194         }
1195
1196         if (dev->data->rx_queues) {
1197                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1198                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1199         }
1200 }
1201
1202 static void
1203 fm10k_dev_close(struct rte_eth_dev *dev)
1204 {
1205         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1206
1207         PMD_INIT_FUNC_TRACE();
1208
1209         fm10k_mbx_lock(hw);
1210         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1211                 MAX_LPORT_NUM, false);
1212         fm10k_mbx_unlock(hw);
1213
1214         /* allow 100ms for device to quiesce */
1215         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1216
1217         /* Stop mailbox service first */
1218         fm10k_close_mbx_service(hw);
1219         fm10k_dev_stop(dev);
1220         fm10k_dev_queue_release(dev);
1221         fm10k_stop_hw(hw);
1222 }
1223
1224 static int
1225 fm10k_link_update(struct rte_eth_dev *dev,
1226         __rte_unused int wait_to_complete)
1227 {
1228         struct fm10k_dev_info *dev_info =
1229                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1230         PMD_INIT_FUNC_TRACE();
1231
1232         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1233         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1234         dev->data->dev_link.link_status =
1235                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1236         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1237
1238         return 0;
1239 }
1240
1241 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1242         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1243 {
1244         unsigned i, q;
1245         unsigned count = 0;
1246
1247         if (xstats_names != NULL) {
1248                 /* Note: limit checked in rte_eth_xstats_names() */
1249
1250                 /* Global stats */
1251                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1252                         snprintf(xstats_names[count].name,
1253                                 sizeof(xstats_names[count].name),
1254                                 "%s", fm10k_hw_stats_strings[count].name);
1255                         count++;
1256                 }
1257
1258                 /* PF queue stats */
1259                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1260                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1261                                 snprintf(xstats_names[count].name,
1262                                         sizeof(xstats_names[count].name),
1263                                         "rx_q%u_%s", q,
1264                                         fm10k_hw_stats_rx_q_strings[i].name);
1265                                 count++;
1266                         }
1267                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1268                                 snprintf(xstats_names[count].name,
1269                                         sizeof(xstats_names[count].name),
1270                                         "tx_q%u_%s", q,
1271                                         fm10k_hw_stats_tx_q_strings[i].name);
1272                                 count++;
1273                         }
1274                 }
1275         }
1276         return FM10K_NB_XSTATS;
1277 }
1278
1279 static int
1280 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1281                  unsigned n)
1282 {
1283         struct fm10k_hw_stats *hw_stats =
1284                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1285         unsigned i, q, count = 0;
1286
1287         if (n < FM10K_NB_XSTATS)
1288                 return FM10K_NB_XSTATS;
1289
1290         /* Global stats */
1291         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1292                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1293                         fm10k_hw_stats_strings[count].offset);
1294                 xstats[count].id = count;
1295                 count++;
1296         }
1297
1298         /* PF queue stats */
1299         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1300                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1301                         xstats[count].value =
1302                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1303                                 fm10k_hw_stats_rx_q_strings[i].offset);
1304                         xstats[count].id = count;
1305                         count++;
1306                 }
1307                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1308                         xstats[count].value =
1309                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1310                                 fm10k_hw_stats_tx_q_strings[i].offset);
1311                         xstats[count].id = count;
1312                         count++;
1313                 }
1314         }
1315
1316         return FM10K_NB_XSTATS;
1317 }
1318
1319 static int
1320 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1321 {
1322         uint64_t ipackets, opackets, ibytes, obytes;
1323         struct fm10k_hw *hw =
1324                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1325         struct fm10k_hw_stats *hw_stats =
1326                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1327         int i;
1328
1329         PMD_INIT_FUNC_TRACE();
1330
1331         fm10k_update_hw_stats(hw, hw_stats);
1332
1333         ipackets = opackets = ibytes = obytes = 0;
1334         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1335                 (i < hw->mac.max_queues); ++i) {
1336                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1337                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1338                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1339                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1340                 ipackets += stats->q_ipackets[i];
1341                 opackets += stats->q_opackets[i];
1342                 ibytes   += stats->q_ibytes[i];
1343                 obytes   += stats->q_obytes[i];
1344         }
1345         stats->ipackets = ipackets;
1346         stats->opackets = opackets;
1347         stats->ibytes = ibytes;
1348         stats->obytes = obytes;
1349         return 0;
1350 }
1351
1352 static void
1353 fm10k_stats_reset(struct rte_eth_dev *dev)
1354 {
1355         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1356         struct fm10k_hw_stats *hw_stats =
1357                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1358
1359         PMD_INIT_FUNC_TRACE();
1360
1361         memset(hw_stats, 0, sizeof(*hw_stats));
1362         fm10k_rebind_hw_stats(hw, hw_stats);
1363 }
1364
1365 static void
1366 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1367         struct rte_eth_dev_info *dev_info)
1368 {
1369         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1370         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1371
1372         PMD_INIT_FUNC_TRACE();
1373
1374         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1375         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1376         dev_info->max_rx_queues      = hw->mac.max_queues;
1377         dev_info->max_tx_queues      = hw->mac.max_queues;
1378         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1379         dev_info->max_hash_mac_addrs = 0;
1380         dev_info->max_vfs            = pdev->max_vfs;
1381         dev_info->vmdq_pool_base     = 0;
1382         dev_info->vmdq_queue_base    = 0;
1383         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1384         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1385         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1386         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1387                                     dev_info->rx_queue_offload_capa;
1388         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1389         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1390                                     dev_info->tx_queue_offload_capa;
1391
1392         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1393         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1394
1395         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1396                 .rx_thresh = {
1397                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1398                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1399                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1400                 },
1401                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1402                 .rx_drop_en = 0,
1403                 .offloads = 0,
1404         };
1405
1406         dev_info->default_txconf = (struct rte_eth_txconf) {
1407                 .tx_thresh = {
1408                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1409                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1410                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1411                 },
1412                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1413                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1414                 .offloads = 0,
1415         };
1416
1417         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1418                 .nb_max = FM10K_MAX_RX_DESC,
1419                 .nb_min = FM10K_MIN_RX_DESC,
1420                 .nb_align = FM10K_MULT_RX_DESC,
1421         };
1422
1423         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1424                 .nb_max = FM10K_MAX_TX_DESC,
1425                 .nb_min = FM10K_MIN_TX_DESC,
1426                 .nb_align = FM10K_MULT_TX_DESC,
1427                 .nb_seg_max = FM10K_TX_MAX_SEG,
1428                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1429         };
1430
1431         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1432                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1433                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1434 }
1435
1436 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1437 static const uint32_t *
1438 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1439 {
1440         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1441             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1442                 static uint32_t ptypes[] = {
1443                         /* refers to rx_desc_to_ol_flags() */
1444                         RTE_PTYPE_L2_ETHER,
1445                         RTE_PTYPE_L3_IPV4,
1446                         RTE_PTYPE_L3_IPV4_EXT,
1447                         RTE_PTYPE_L3_IPV6,
1448                         RTE_PTYPE_L3_IPV6_EXT,
1449                         RTE_PTYPE_L4_TCP,
1450                         RTE_PTYPE_L4_UDP,
1451                         RTE_PTYPE_UNKNOWN
1452                 };
1453
1454                 return ptypes;
1455         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1456                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1457                 static uint32_t ptypes_vec[] = {
1458                         /* refers to fm10k_desc_to_pktype_v() */
1459                         RTE_PTYPE_L3_IPV4,
1460                         RTE_PTYPE_L3_IPV4_EXT,
1461                         RTE_PTYPE_L3_IPV6,
1462                         RTE_PTYPE_L3_IPV6_EXT,
1463                         RTE_PTYPE_L4_TCP,
1464                         RTE_PTYPE_L4_UDP,
1465                         RTE_PTYPE_TUNNEL_GENEVE,
1466                         RTE_PTYPE_TUNNEL_NVGRE,
1467                         RTE_PTYPE_TUNNEL_VXLAN,
1468                         RTE_PTYPE_TUNNEL_GRE,
1469                         RTE_PTYPE_UNKNOWN
1470                 };
1471
1472                 return ptypes_vec;
1473         }
1474
1475         return NULL;
1476 }
1477 #else
1478 static const uint32_t *
1479 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1480 {
1481         return NULL;
1482 }
1483 #endif
1484
1485 static int
1486 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1487 {
1488         s32 result;
1489         uint16_t mac_num = 0;
1490         uint32_t vid_idx, vid_bit, mac_index;
1491         struct fm10k_hw *hw;
1492         struct fm10k_macvlan_filter_info *macvlan;
1493         struct rte_eth_dev_data *data = dev->data;
1494
1495         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1496         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1497
1498         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1499                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1500                 return -EINVAL;
1501         }
1502
1503         if (vlan_id > ETH_VLAN_ID_MAX) {
1504                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1505                 return -EINVAL;
1506         }
1507
1508         vid_idx = FM10K_VFTA_IDX(vlan_id);
1509         vid_bit = FM10K_VFTA_BIT(vlan_id);
1510         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1511         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1512                 return 0;
1513         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1514         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1515                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1516                         "in the VLAN filter table");
1517                 return -EINVAL;
1518         }
1519
1520         fm10k_mbx_lock(hw);
1521         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1522         fm10k_mbx_unlock(hw);
1523         if (result != FM10K_SUCCESS) {
1524                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1525                 return -EIO;
1526         }
1527
1528         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1529                         (result == FM10K_SUCCESS); mac_index++) {
1530                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1531                         continue;
1532                 if (mac_num > macvlan->mac_num - 1) {
1533                         PMD_INIT_LOG(ERR, "MAC address number "
1534                                         "not match");
1535                         break;
1536                 }
1537                 fm10k_mbx_lock(hw);
1538                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1539                         data->mac_addrs[mac_index].addr_bytes,
1540                         vlan_id, on, 0);
1541                 fm10k_mbx_unlock(hw);
1542                 mac_num++;
1543         }
1544         if (result != FM10K_SUCCESS) {
1545                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1546                 return -EIO;
1547         }
1548
1549         if (on) {
1550                 macvlan->vlan_num++;
1551                 macvlan->vfta[vid_idx] |= vid_bit;
1552         } else {
1553                 macvlan->vlan_num--;
1554                 macvlan->vfta[vid_idx] &= ~vid_bit;
1555         }
1556         return 0;
1557 }
1558
1559 static int
1560 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1561 {
1562         if (mask & ETH_VLAN_STRIP_MASK) {
1563                 if (!(dev->data->dev_conf.rxmode.offloads &
1564                         DEV_RX_OFFLOAD_VLAN_STRIP))
1565                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1566                                         "always on in fm10k");
1567         }
1568
1569         if (mask & ETH_VLAN_EXTEND_MASK) {
1570                 if (dev->data->dev_conf.rxmode.offloads &
1571                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1572                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1573                                         "supported in fm10k");
1574         }
1575
1576         if (mask & ETH_VLAN_FILTER_MASK) {
1577                 if (!(dev->data->dev_conf.rxmode.offloads &
1578                         DEV_RX_OFFLOAD_VLAN_FILTER))
1579                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1580         }
1581
1582         return 0;
1583 }
1584
1585 /* Add/Remove a MAC address, and update filters to main VSI */
1586 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1587                 const u8 *mac, bool add, uint32_t pool)
1588 {
1589         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1590         struct fm10k_macvlan_filter_info *macvlan;
1591         uint32_t i, j, k;
1592
1593         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1594
1595         if (pool != MAIN_VSI_POOL_NUMBER) {
1596                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1597                         "mac to pool %u", pool);
1598                 return;
1599         }
1600         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1601                 if (!macvlan->vfta[j])
1602                         continue;
1603                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1604                         if (!(macvlan->vfta[j] & (1 << k)))
1605                                 continue;
1606                         if (i + 1 > macvlan->vlan_num) {
1607                                 PMD_INIT_LOG(ERR, "vlan number not match");
1608                                 return;
1609                         }
1610                         fm10k_mbx_lock(hw);
1611                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1612                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1613                         fm10k_mbx_unlock(hw);
1614                         i++;
1615                 }
1616         }
1617 }
1618
1619 /* Add/Remove a MAC address, and update filters to VMDQ */
1620 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1621                 const u8 *mac, bool add, uint32_t pool)
1622 {
1623         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1624         struct fm10k_macvlan_filter_info *macvlan;
1625         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1626         uint32_t i;
1627
1628         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1629         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1630
1631         if (pool > macvlan->nb_queue_pools) {
1632                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1633                         " Max pool is %u",
1634                         pool, macvlan->nb_queue_pools);
1635                 return;
1636         }
1637         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1638                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1639                         continue;
1640                 fm10k_mbx_lock(hw);
1641                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1642                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1643                 fm10k_mbx_unlock(hw);
1644         }
1645 }
1646
1647 /* Add/Remove a MAC address, and update filters */
1648 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1649                 const u8 *mac, bool add, uint32_t pool)
1650 {
1651         struct fm10k_macvlan_filter_info *macvlan;
1652
1653         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1654
1655         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1656                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1657         else
1658                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1659
1660         if (add)
1661                 macvlan->mac_num++;
1662         else
1663                 macvlan->mac_num--;
1664 }
1665
1666 /* Add a MAC address, and update filters */
1667 static int
1668 fm10k_macaddr_add(struct rte_eth_dev *dev,
1669                 struct ether_addr *mac_addr,
1670                 uint32_t index,
1671                 uint32_t pool)
1672 {
1673         struct fm10k_macvlan_filter_info *macvlan;
1674
1675         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1676         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1677         macvlan->mac_vmdq_id[index] = pool;
1678         return 0;
1679 }
1680
1681 /* Remove a MAC address, and update filters */
1682 static void
1683 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1684 {
1685         struct rte_eth_dev_data *data = dev->data;
1686         struct fm10k_macvlan_filter_info *macvlan;
1687
1688         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1689         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1690                         FALSE, macvlan->mac_vmdq_id[index]);
1691         macvlan->mac_vmdq_id[index] = 0;
1692 }
1693
1694 static inline int
1695 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1696 {
1697         if ((request < min) || (request > max) || ((request % mult) != 0))
1698                 return -1;
1699         else
1700                 return 0;
1701 }
1702
1703
1704 static inline int
1705 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1706 {
1707         if ((request < min) || (request > max) || ((div % request) != 0))
1708                 return -1;
1709         else
1710                 return 0;
1711 }
1712
1713 static inline int
1714 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1715 {
1716         uint16_t rx_free_thresh;
1717
1718         if (conf->rx_free_thresh == 0)
1719                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1720         else
1721                 rx_free_thresh = conf->rx_free_thresh;
1722
1723         /* make sure the requested threshold satisfies the constraints */
1724         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1725                         FM10K_RX_FREE_THRESH_MAX(q),
1726                         FM10K_RX_FREE_THRESH_DIV(q),
1727                         rx_free_thresh)) {
1728                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1729                         "less than or equal to %u, "
1730                         "greater than or equal to %u, "
1731                         "and a divisor of %u",
1732                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1733                         FM10K_RX_FREE_THRESH_MIN(q),
1734                         FM10K_RX_FREE_THRESH_DIV(q));
1735                 return -EINVAL;
1736         }
1737
1738         q->alloc_thresh = rx_free_thresh;
1739         q->drop_en = conf->rx_drop_en;
1740         q->rx_deferred_start = conf->rx_deferred_start;
1741
1742         return 0;
1743 }
1744
1745 /*
1746  * Hardware requires specific alignment for Rx packet buffers. At
1747  * least one of the following two conditions must be satisfied.
1748  *  1. Address is 512B aligned
1749  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1750  *
1751  * As such, the driver may need to adjust the DMA address within the
1752  * buffer by up to 512B.
1753  *
1754  * return 1 if the element size is valid, otherwise return 0.
1755  */
1756 static int
1757 mempool_element_size_valid(struct rte_mempool *mp)
1758 {
1759         uint32_t min_size;
1760
1761         /* elt_size includes mbuf header and headroom */
1762         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1763                         RTE_PKTMBUF_HEADROOM;
1764
1765         /* account for up to 512B of alignment */
1766         min_size -= FM10K_RX_DATABUF_ALIGN;
1767
1768         /* sanity check for overflow */
1769         if (min_size > mp->elt_size)
1770                 return 0;
1771
1772         /* size is valid */
1773         return 1;
1774 }
1775
1776 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1777 {
1778         RTE_SET_USED(dev);
1779
1780         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1781 }
1782
1783 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1784 {
1785         RTE_SET_USED(dev);
1786
1787         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1788                            DEV_RX_OFFLOAD_VLAN_FILTER |
1789                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1790                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1791                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1792                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1793                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1794 }
1795
1796 static int
1797 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1798         uint16_t nb_desc, unsigned int socket_id,
1799         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1800 {
1801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1802         struct fm10k_dev_info *dev_info =
1803                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1804         struct fm10k_rx_queue *q;
1805         const struct rte_memzone *mz;
1806         uint64_t offloads;
1807
1808         PMD_INIT_FUNC_TRACE();
1809
1810         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1811
1812         /* make sure the mempool element size can account for alignment. */
1813         if (!mempool_element_size_valid(mp)) {
1814                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1815                 return -EINVAL;
1816         }
1817
1818         /* make sure a valid number of descriptors have been requested */
1819         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1820                                 FM10K_MULT_RX_DESC, nb_desc)) {
1821                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1822                         "less than or equal to %"PRIu32", "
1823                         "greater than or equal to %u, "
1824                         "and a multiple of %u",
1825                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1826                         FM10K_MULT_RX_DESC);
1827                 return -EINVAL;
1828         }
1829
1830         /*
1831          * if this queue existed already, free the associated memory. The
1832          * queue cannot be reused in case we need to allocate memory on
1833          * different socket than was previously used.
1834          */
1835         if (dev->data->rx_queues[queue_id] != NULL) {
1836                 rx_queue_free(dev->data->rx_queues[queue_id]);
1837                 dev->data->rx_queues[queue_id] = NULL;
1838         }
1839
1840         /* allocate memory for the queue structure */
1841         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1842                                 socket_id);
1843         if (q == NULL) {
1844                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1845                 return -ENOMEM;
1846         }
1847
1848         /* setup queue */
1849         q->mp = mp;
1850         q->nb_desc = nb_desc;
1851         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1852         q->port_id = dev->data->port_id;
1853         q->queue_id = queue_id;
1854         q->tail_ptr = (volatile uint32_t *)
1855                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1856         q->offloads = offloads;
1857         if (handle_rxconf(q, conf))
1858                 return -EINVAL;
1859
1860         /* allocate memory for the software ring */
1861         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1862                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1863                         RTE_CACHE_LINE_SIZE, socket_id);
1864         if (q->sw_ring == NULL) {
1865                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1866                 rte_free(q);
1867                 return -ENOMEM;
1868         }
1869
1870         /*
1871          * allocate memory for the hardware descriptor ring. A memzone large
1872          * enough to hold the maximum ring size is requested to allow for
1873          * resizing in later calls to the queue setup function.
1874          */
1875         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1876                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1877                                       socket_id);
1878         if (mz == NULL) {
1879                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1880                 rte_free(q->sw_ring);
1881                 rte_free(q);
1882                 return -ENOMEM;
1883         }
1884         q->hw_ring = mz->addr;
1885         q->hw_ring_phys_addr = mz->iova;
1886
1887         /* Check if number of descs satisfied Vector requirement */
1888         if (!rte_is_power_of_2(nb_desc)) {
1889                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1890                                     "preconditions - canceling the feature for "
1891                                     "the whole port[%d]",
1892                              q->queue_id, q->port_id);
1893                 dev_info->rx_vec_allowed = false;
1894         } else
1895                 fm10k_rxq_vec_setup(q);
1896
1897         dev->data->rx_queues[queue_id] = q;
1898         return 0;
1899 }
1900
1901 static void
1902 fm10k_rx_queue_release(void *queue)
1903 {
1904         PMD_INIT_FUNC_TRACE();
1905
1906         rx_queue_free(queue);
1907 }
1908
1909 static inline int
1910 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1911 {
1912         uint16_t tx_free_thresh;
1913         uint16_t tx_rs_thresh;
1914
1915         /* constraint MACROs require that tx_free_thresh is configured
1916          * before tx_rs_thresh */
1917         if (conf->tx_free_thresh == 0)
1918                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1919         else
1920                 tx_free_thresh = conf->tx_free_thresh;
1921
1922         /* make sure the requested threshold satisfies the constraints */
1923         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1924                         FM10K_TX_FREE_THRESH_MAX(q),
1925                         FM10K_TX_FREE_THRESH_DIV(q),
1926                         tx_free_thresh)) {
1927                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1928                         "less than or equal to %u, "
1929                         "greater than or equal to %u, "
1930                         "and a divisor of %u",
1931                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1932                         FM10K_TX_FREE_THRESH_MIN(q),
1933                         FM10K_TX_FREE_THRESH_DIV(q));
1934                 return -EINVAL;
1935         }
1936
1937         q->free_thresh = tx_free_thresh;
1938
1939         if (conf->tx_rs_thresh == 0)
1940                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1941         else
1942                 tx_rs_thresh = conf->tx_rs_thresh;
1943
1944         q->tx_deferred_start = conf->tx_deferred_start;
1945
1946         /* make sure the requested threshold satisfies the constraints */
1947         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1948                         FM10K_TX_RS_THRESH_MAX(q),
1949                         FM10K_TX_RS_THRESH_DIV(q),
1950                         tx_rs_thresh)) {
1951                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1952                         "less than or equal to %u, "
1953                         "greater than or equal to %u, "
1954                         "and a divisor of %u",
1955                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1956                         FM10K_TX_RS_THRESH_MIN(q),
1957                         FM10K_TX_RS_THRESH_DIV(q));
1958                 return -EINVAL;
1959         }
1960
1961         q->rs_thresh = tx_rs_thresh;
1962
1963         return 0;
1964 }
1965
1966 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1967 {
1968         RTE_SET_USED(dev);
1969
1970         return 0;
1971 }
1972
1973 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1974 {
1975         RTE_SET_USED(dev);
1976
1977         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1978                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1979                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1980                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1981                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1982                           DEV_TX_OFFLOAD_TCP_TSO);
1983 }
1984
1985 static int
1986 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1987         uint16_t nb_desc, unsigned int socket_id,
1988         const struct rte_eth_txconf *conf)
1989 {
1990         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1991         struct fm10k_tx_queue *q;
1992         const struct rte_memzone *mz;
1993         uint64_t offloads;
1994
1995         PMD_INIT_FUNC_TRACE();
1996
1997         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
1998
1999         /* make sure a valid number of descriptors have been requested */
2000         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2001                                 FM10K_MULT_TX_DESC, nb_desc)) {
2002                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2003                         "less than or equal to %"PRIu32", "
2004                         "greater than or equal to %u, "
2005                         "and a multiple of %u",
2006                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2007                         FM10K_MULT_TX_DESC);
2008                 return -EINVAL;
2009         }
2010
2011         /*
2012          * if this queue existed already, free the associated memory. The
2013          * queue cannot be reused in case we need to allocate memory on
2014          * different socket than was previously used.
2015          */
2016         if (dev->data->tx_queues[queue_id] != NULL) {
2017                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2018
2019                 tx_queue_free(txq);
2020                 dev->data->tx_queues[queue_id] = NULL;
2021         }
2022
2023         /* allocate memory for the queue structure */
2024         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2025                                 socket_id);
2026         if (q == NULL) {
2027                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2028                 return -ENOMEM;
2029         }
2030
2031         /* setup queue */
2032         q->nb_desc = nb_desc;
2033         q->port_id = dev->data->port_id;
2034         q->queue_id = queue_id;
2035         q->offloads = offloads;
2036         q->ops = &def_txq_ops;
2037         q->tail_ptr = (volatile uint32_t *)
2038                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2039         if (handle_txconf(q, conf))
2040                 return -EINVAL;
2041
2042         /* allocate memory for the software ring */
2043         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2044                                         nb_desc * sizeof(struct rte_mbuf *),
2045                                         RTE_CACHE_LINE_SIZE, socket_id);
2046         if (q->sw_ring == NULL) {
2047                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2048                 rte_free(q);
2049                 return -ENOMEM;
2050         }
2051
2052         /*
2053          * allocate memory for the hardware descriptor ring. A memzone large
2054          * enough to hold the maximum ring size is requested to allow for
2055          * resizing in later calls to the queue setup function.
2056          */
2057         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2058                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2059                                       socket_id);
2060         if (mz == NULL) {
2061                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2062                 rte_free(q->sw_ring);
2063                 rte_free(q);
2064                 return -ENOMEM;
2065         }
2066         q->hw_ring = mz->addr;
2067         q->hw_ring_phys_addr = mz->iova;
2068
2069         /*
2070          * allocate memory for the RS bit tracker. Enough slots to hold the
2071          * descriptor index for each RS bit needing to be set are required.
2072          */
2073         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2074                                 ((nb_desc + 1) / q->rs_thresh) *
2075                                 sizeof(uint16_t),
2076                                 RTE_CACHE_LINE_SIZE, socket_id);
2077         if (q->rs_tracker.list == NULL) {
2078                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2079                 rte_free(q->sw_ring);
2080                 rte_free(q);
2081                 return -ENOMEM;
2082         }
2083
2084         dev->data->tx_queues[queue_id] = q;
2085         return 0;
2086 }
2087
2088 static void
2089 fm10k_tx_queue_release(void *queue)
2090 {
2091         struct fm10k_tx_queue *q = queue;
2092         PMD_INIT_FUNC_TRACE();
2093
2094         tx_queue_free(q);
2095 }
2096
2097 static int
2098 fm10k_reta_update(struct rte_eth_dev *dev,
2099                         struct rte_eth_rss_reta_entry64 *reta_conf,
2100                         uint16_t reta_size)
2101 {
2102         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2103         uint16_t i, j, idx, shift;
2104         uint8_t mask;
2105         uint32_t reta;
2106
2107         PMD_INIT_FUNC_TRACE();
2108
2109         if (reta_size > FM10K_MAX_RSS_INDICES) {
2110                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2111                         "(%d) doesn't match the number hardware can supported "
2112                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2113                 return -EINVAL;
2114         }
2115
2116         /*
2117          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2118          * 128-entries in 32 registers
2119          */
2120         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2121                 idx = i / RTE_RETA_GROUP_SIZE;
2122                 shift = i % RTE_RETA_GROUP_SIZE;
2123                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2124                                 BIT_MASK_PER_UINT32);
2125                 if (mask == 0)
2126                         continue;
2127
2128                 reta = 0;
2129                 if (mask != BIT_MASK_PER_UINT32)
2130                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2131
2132                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2133                         if (mask & (0x1 << j)) {
2134                                 if (mask != 0xF)
2135                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2136                                 reta |= reta_conf[idx].reta[shift + j] <<
2137                                                 (CHAR_BIT * j);
2138                         }
2139                 }
2140                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2141         }
2142
2143         return 0;
2144 }
2145
2146 static int
2147 fm10k_reta_query(struct rte_eth_dev *dev,
2148                         struct rte_eth_rss_reta_entry64 *reta_conf,
2149                         uint16_t reta_size)
2150 {
2151         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2152         uint16_t i, j, idx, shift;
2153         uint8_t mask;
2154         uint32_t reta;
2155
2156         PMD_INIT_FUNC_TRACE();
2157
2158         if (reta_size < FM10K_MAX_RSS_INDICES) {
2159                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2160                         "(%d) doesn't match the number hardware can supported "
2161                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2162                 return -EINVAL;
2163         }
2164
2165         /*
2166          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2167          * 128-entries in 32 registers
2168          */
2169         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2170                 idx = i / RTE_RETA_GROUP_SIZE;
2171                 shift = i % RTE_RETA_GROUP_SIZE;
2172                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2173                                 BIT_MASK_PER_UINT32);
2174                 if (mask == 0)
2175                         continue;
2176
2177                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2178                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2179                         if (mask & (0x1 << j))
2180                                 reta_conf[idx].reta[shift + j] = ((reta >>
2181                                         CHAR_BIT * j) & UINT8_MAX);
2182                 }
2183         }
2184
2185         return 0;
2186 }
2187
2188 static int
2189 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2190         struct rte_eth_rss_conf *rss_conf)
2191 {
2192         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2193         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2194         uint32_t mrqc;
2195         uint64_t hf = rss_conf->rss_hf;
2196         int i;
2197
2198         PMD_INIT_FUNC_TRACE();
2199
2200         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2201                                 FM10K_RSSRK_ENTRIES_PER_REG))
2202                 return -EINVAL;
2203
2204         if (hf == 0)
2205                 return -EINVAL;
2206
2207         mrqc = 0;
2208         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2209         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2210         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2211         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2212         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2213         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2214         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2215         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2216         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2217
2218         /* If the mapping doesn't fit any supported, return */
2219         if (mrqc == 0)
2220                 return -EINVAL;
2221
2222         if (key != NULL)
2223                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2224                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2225
2226         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2227
2228         return 0;
2229 }
2230
2231 static int
2232 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2233         struct rte_eth_rss_conf *rss_conf)
2234 {
2235         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2236         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2237         uint32_t mrqc;
2238         uint64_t hf;
2239         int i;
2240
2241         PMD_INIT_FUNC_TRACE();
2242
2243         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2244                                 FM10K_RSSRK_ENTRIES_PER_REG))
2245                 return -EINVAL;
2246
2247         if (key != NULL)
2248                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2249                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2250
2251         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2252         hf = 0;
2253         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2254         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2255         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2256         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2257         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2258         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2259         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2260         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2261         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2262
2263         rss_conf->rss_hf = hf;
2264
2265         return 0;
2266 }
2267
2268 static void
2269 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2270 {
2271         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2272         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2273
2274         /* Bind all local non-queue interrupt to vector 0 */
2275         int_map |= FM10K_MISC_VEC_ID;
2276
2277         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2278         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2279         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2280         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2281         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2282         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2283
2284         /* Enable misc causes */
2285         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2286                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2287                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2288                                 FM10K_EIMR_ENABLE(MAILBOX) |
2289                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2290                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2291                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2292                                 FM10K_EIMR_ENABLE(VFLR));
2293
2294         /* Enable ITR 0 */
2295         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2296                                         FM10K_ITR_MASK_CLEAR);
2297         FM10K_WRITE_FLUSH(hw);
2298 }
2299
2300 static void
2301 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2302 {
2303         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2304         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2305
2306         int_map |= FM10K_MISC_VEC_ID;
2307
2308         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2309         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2310         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2311         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2312         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2313         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2314
2315         /* Disable misc causes */
2316         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2317                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2318                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2319                                 FM10K_EIMR_DISABLE(MAILBOX) |
2320                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2321                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2322                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2323                                 FM10K_EIMR_DISABLE(VFLR));
2324
2325         /* Disable ITR 0 */
2326         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2327         FM10K_WRITE_FLUSH(hw);
2328 }
2329
2330 static void
2331 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2332 {
2333         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2334         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2335
2336         /* Bind all local non-queue interrupt to vector 0 */
2337         int_map |= FM10K_MISC_VEC_ID;
2338
2339         /* Only INT 0 available, other 15 are reserved. */
2340         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2341
2342         /* Enable ITR 0 */
2343         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2344                                         FM10K_ITR_MASK_CLEAR);
2345         FM10K_WRITE_FLUSH(hw);
2346 }
2347
2348 static void
2349 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2350 {
2351         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2352         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2353
2354         int_map |= FM10K_MISC_VEC_ID;
2355
2356         /* Only INT 0 available, other 15 are reserved. */
2357         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2358
2359         /* Disable ITR 0 */
2360         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2361         FM10K_WRITE_FLUSH(hw);
2362 }
2363
2364 static int
2365 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2366 {
2367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2368         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2369
2370         /* Enable ITR */
2371         if (hw->mac.type == fm10k_mac_pf)
2372                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2373                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2374         else
2375                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2376                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2377         rte_intr_enable(&pdev->intr_handle);
2378         return 0;
2379 }
2380
2381 static int
2382 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2383 {
2384         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2385         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2386
2387         /* Disable ITR */
2388         if (hw->mac.type == fm10k_mac_pf)
2389                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2390                         FM10K_ITR_MASK_SET);
2391         else
2392                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2393                         FM10K_ITR_MASK_SET);
2394         return 0;
2395 }
2396
2397 static int
2398 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2399 {
2400         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2401         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2402         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2403         uint32_t intr_vector, vec;
2404         uint16_t queue_id;
2405         int result = 0;
2406
2407         /* fm10k needs one separate interrupt for mailbox,
2408          * so only drivers which support multiple interrupt vectors
2409          * e.g. vfio-pci can work for fm10k interrupt mode
2410          */
2411         if (!rte_intr_cap_multiple(intr_handle) ||
2412                         dev->data->dev_conf.intr_conf.rxq == 0)
2413                 return result;
2414
2415         intr_vector = dev->data->nb_rx_queues;
2416
2417         /* disable interrupt first */
2418         rte_intr_disable(intr_handle);
2419         if (hw->mac.type == fm10k_mac_pf)
2420                 fm10k_dev_disable_intr_pf(dev);
2421         else
2422                 fm10k_dev_disable_intr_vf(dev);
2423
2424         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2425                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2426                 result = -EIO;
2427         }
2428
2429         if (rte_intr_dp_is_en(intr_handle) && !result) {
2430                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2431                         dev->data->nb_rx_queues * sizeof(int), 0);
2432                 if (intr_handle->intr_vec) {
2433                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2434                                         queue_id < dev->data->nb_rx_queues;
2435                                         queue_id++) {
2436                                 intr_handle->intr_vec[queue_id] = vec;
2437                                 if (vec < intr_handle->nb_efd - 1
2438                                                 + FM10K_RX_VEC_START)
2439                                         vec++;
2440                         }
2441                 } else {
2442                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2443                                 " intr_vec", dev->data->nb_rx_queues);
2444                         rte_intr_efd_disable(intr_handle);
2445                         result = -ENOMEM;
2446                 }
2447         }
2448
2449         if (hw->mac.type == fm10k_mac_pf)
2450                 fm10k_dev_enable_intr_pf(dev);
2451         else
2452                 fm10k_dev_enable_intr_vf(dev);
2453         rte_intr_enable(intr_handle);
2454         hw->mac.ops.update_int_moderator(hw);
2455         return result;
2456 }
2457
2458 static int
2459 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2460 {
2461         struct fm10k_fault fault;
2462         int err;
2463         const char *estr = "Unknown error";
2464
2465         /* Process PCA fault */
2466         if (eicr & FM10K_EICR_PCA_FAULT) {
2467                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2468                 if (err)
2469                         goto error;
2470                 switch (fault.type) {
2471                 case PCA_NO_FAULT:
2472                         estr = "PCA_NO_FAULT"; break;
2473                 case PCA_UNMAPPED_ADDR:
2474                         estr = "PCA_UNMAPPED_ADDR"; break;
2475                 case PCA_BAD_QACCESS_PF:
2476                         estr = "PCA_BAD_QACCESS_PF"; break;
2477                 case PCA_BAD_QACCESS_VF:
2478                         estr = "PCA_BAD_QACCESS_VF"; break;
2479                 case PCA_MALICIOUS_REQ:
2480                         estr = "PCA_MALICIOUS_REQ"; break;
2481                 case PCA_POISONED_TLP:
2482                         estr = "PCA_POISONED_TLP"; break;
2483                 case PCA_TLP_ABORT:
2484                         estr = "PCA_TLP_ABORT"; break;
2485                 default:
2486                         goto error;
2487                 }
2488                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2489                         estr, fault.func ? "VF" : "PF", fault.func,
2490                         fault.address, fault.specinfo);
2491         }
2492
2493         /* Process THI fault */
2494         if (eicr & FM10K_EICR_THI_FAULT) {
2495                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2496                 if (err)
2497                         goto error;
2498                 switch (fault.type) {
2499                 case THI_NO_FAULT:
2500                         estr = "THI_NO_FAULT"; break;
2501                 case THI_MAL_DIS_Q_FAULT:
2502                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2503                 default:
2504                         goto error;
2505                 }
2506                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2507                         estr, fault.func ? "VF" : "PF", fault.func,
2508                         fault.address, fault.specinfo);
2509         }
2510
2511         /* Process FUM fault */
2512         if (eicr & FM10K_EICR_FUM_FAULT) {
2513                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2514                 if (err)
2515                         goto error;
2516                 switch (fault.type) {
2517                 case FUM_NO_FAULT:
2518                         estr = "FUM_NO_FAULT"; break;
2519                 case FUM_UNMAPPED_ADDR:
2520                         estr = "FUM_UNMAPPED_ADDR"; break;
2521                 case FUM_POISONED_TLP:
2522                         estr = "FUM_POISONED_TLP"; break;
2523                 case FUM_BAD_VF_QACCESS:
2524                         estr = "FUM_BAD_VF_QACCESS"; break;
2525                 case FUM_ADD_DECODE_ERR:
2526                         estr = "FUM_ADD_DECODE_ERR"; break;
2527                 case FUM_RO_ERROR:
2528                         estr = "FUM_RO_ERROR"; break;
2529                 case FUM_QPRC_CRC_ERROR:
2530                         estr = "FUM_QPRC_CRC_ERROR"; break;
2531                 case FUM_CSR_TIMEOUT:
2532                         estr = "FUM_CSR_TIMEOUT"; break;
2533                 case FUM_INVALID_TYPE:
2534                         estr = "FUM_INVALID_TYPE"; break;
2535                 case FUM_INVALID_LENGTH:
2536                         estr = "FUM_INVALID_LENGTH"; break;
2537                 case FUM_INVALID_BE:
2538                         estr = "FUM_INVALID_BE"; break;
2539                 case FUM_INVALID_ALIGN:
2540                         estr = "FUM_INVALID_ALIGN"; break;
2541                 default:
2542                         goto error;
2543                 }
2544                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2545                         estr, fault.func ? "VF" : "PF", fault.func,
2546                         fault.address, fault.specinfo);
2547         }
2548
2549         return 0;
2550 error:
2551         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2552         return err;
2553 }
2554
2555 /**
2556  * PF interrupt handler triggered by NIC for handling specific interrupt.
2557  *
2558  * @param handle
2559  *  Pointer to interrupt handle.
2560  * @param param
2561  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2562  *
2563  * @return
2564  *  void
2565  */
2566 static void
2567 fm10k_dev_interrupt_handler_pf(void *param)
2568 {
2569         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2570         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2571         uint32_t cause, status;
2572         struct fm10k_dev_info *dev_info =
2573                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2574         int status_mbx;
2575         s32 err;
2576
2577         if (hw->mac.type != fm10k_mac_pf)
2578                 return;
2579
2580         cause = FM10K_READ_REG(hw, FM10K_EICR);
2581
2582         /* Handle PCI fault cases */
2583         if (cause & FM10K_EICR_FAULT_MASK) {
2584                 PMD_INIT_LOG(ERR, "INT: find fault!");
2585                 fm10k_dev_handle_fault(hw, cause);
2586         }
2587
2588         /* Handle switch up/down */
2589         if (cause & FM10K_EICR_SWITCHNOTREADY)
2590                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2591
2592         if (cause & FM10K_EICR_SWITCHREADY) {
2593                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2594                 if (dev_info->sm_down == 1) {
2595                         fm10k_mbx_lock(hw);
2596
2597                         /* For recreating logical ports */
2598                         status_mbx = hw->mac.ops.update_lport_state(hw,
2599                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2600                         if (status_mbx == FM10K_SUCCESS)
2601                                 PMD_INIT_LOG(INFO,
2602                                         "INT: Recreated Logical port");
2603                         else
2604                                 PMD_INIT_LOG(INFO,
2605                                         "INT: Logical ports weren't recreated");
2606
2607                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2608                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2609                         if (status_mbx != FM10K_SUCCESS)
2610                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2611
2612                         fm10k_mbx_unlock(hw);
2613
2614                         /* first clear the internal SW recording structure */
2615                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2616                                                 ETH_MQ_RX_VMDQ_FLAG))
2617                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2618                                         false);
2619
2620                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2621                                         MAIN_VSI_POOL_NUMBER);
2622
2623                         /*
2624                          * Add default mac address and vlan for the logical
2625                          * ports that have been created, leave to the
2626                          * application to fully recover Rx filtering.
2627                          */
2628                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2629                                         MAIN_VSI_POOL_NUMBER);
2630
2631                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2632                                                 ETH_MQ_RX_VMDQ_FLAG))
2633                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2634                                         true);
2635
2636                         dev_info->sm_down = 0;
2637                         _rte_eth_dev_callback_process(dev,
2638                                         RTE_ETH_EVENT_INTR_LSC,
2639                                         NULL);
2640                 }
2641         }
2642
2643         /* Handle mailbox message */
2644         fm10k_mbx_lock(hw);
2645         err = hw->mbx.ops.process(hw, &hw->mbx);
2646         fm10k_mbx_unlock(hw);
2647
2648         if (err == FM10K_ERR_RESET_REQUESTED) {
2649                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2650                 dev_info->sm_down = 1;
2651                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2652                                 NULL);
2653         }
2654
2655         /* Handle SRAM error */
2656         if (cause & FM10K_EICR_SRAMERROR) {
2657                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2658
2659                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2660                 /* Write to clear pending bits */
2661                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2662
2663                 /* Todo: print out error message after shared code  updates */
2664         }
2665
2666         /* Clear these 3 events if having any */
2667         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2668                  FM10K_EICR_SWITCHREADY;
2669         if (cause)
2670                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2671
2672         /* Re-enable interrupt from device side */
2673         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2674                                         FM10K_ITR_MASK_CLEAR);
2675         /* Re-enable interrupt from host side */
2676         rte_intr_enable(dev->intr_handle);
2677 }
2678
2679 /**
2680  * VF interrupt handler triggered by NIC for handling specific interrupt.
2681  *
2682  * @param handle
2683  *  Pointer to interrupt handle.
2684  * @param param
2685  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2686  *
2687  * @return
2688  *  void
2689  */
2690 static void
2691 fm10k_dev_interrupt_handler_vf(void *param)
2692 {
2693         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2694         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2695         struct fm10k_mbx_info *mbx = &hw->mbx;
2696         struct fm10k_dev_info *dev_info =
2697                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2698         const enum fm10k_mbx_state state = mbx->state;
2699         int status_mbx;
2700
2701         if (hw->mac.type != fm10k_mac_vf)
2702                 return;
2703
2704         /* Handle mailbox message if lock is acquired */
2705         fm10k_mbx_lock(hw);
2706         hw->mbx.ops.process(hw, &hw->mbx);
2707         fm10k_mbx_unlock(hw);
2708
2709         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2710                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2711
2712                 fm10k_mbx_lock(hw);
2713                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2714                                 MAX_LPORT_NUM, 1);
2715                 fm10k_mbx_unlock(hw);
2716
2717                 /* Setting reset flag */
2718                 dev_info->sm_down = 1;
2719                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2720                                 NULL);
2721         }
2722
2723         if (dev_info->sm_down == 1 &&
2724                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2725                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2726                 fm10k_mbx_lock(hw);
2727                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2728                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2729                 if (status_mbx != FM10K_SUCCESS)
2730                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2731                 fm10k_mbx_unlock(hw);
2732
2733                 /* first clear the internal SW recording structure */
2734                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2735                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2736                                 MAIN_VSI_POOL_NUMBER);
2737
2738                 /*
2739                  * Add default mac address and vlan for the logical ports that
2740                  * have been created, leave to the application to fully recover
2741                  * Rx filtering.
2742                  */
2743                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2744                                 MAIN_VSI_POOL_NUMBER);
2745                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2746
2747                 dev_info->sm_down = 0;
2748                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2749                                 NULL);
2750         }
2751
2752         /* Re-enable interrupt from device side */
2753         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2754                                         FM10K_ITR_MASK_CLEAR);
2755         /* Re-enable interrupt from host side */
2756         rte_intr_enable(dev->intr_handle);
2757 }
2758
2759 /* Mailbox message handler in VF */
2760 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2761         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2762         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2763         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2764         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2765 };
2766
2767 static int
2768 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2769 {
2770         int err = 0;
2771
2772         /* Initialize mailbox lock */
2773         fm10k_mbx_initlock(hw);
2774
2775         /* Replace default message handler with new ones */
2776         if (hw->mac.type == fm10k_mac_vf)
2777                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2778
2779         if (err) {
2780                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2781                                 err);
2782                 return err;
2783         }
2784         /* Connect to SM for PF device or PF for VF device */
2785         return hw->mbx.ops.connect(hw, &hw->mbx);
2786 }
2787
2788 static void
2789 fm10k_close_mbx_service(struct fm10k_hw *hw)
2790 {
2791         /* Disconnect from SM for PF device or PF for VF device */
2792         hw->mbx.ops.disconnect(hw, &hw->mbx);
2793 }
2794
2795 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2796         .dev_configure          = fm10k_dev_configure,
2797         .dev_start              = fm10k_dev_start,
2798         .dev_stop               = fm10k_dev_stop,
2799         .dev_close              = fm10k_dev_close,
2800         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2801         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2802         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2803         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2804         .stats_get              = fm10k_stats_get,
2805         .xstats_get             = fm10k_xstats_get,
2806         .xstats_get_names       = fm10k_xstats_get_names,
2807         .stats_reset            = fm10k_stats_reset,
2808         .xstats_reset           = fm10k_stats_reset,
2809         .link_update            = fm10k_link_update,
2810         .dev_infos_get          = fm10k_dev_infos_get,
2811         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2812         .vlan_filter_set        = fm10k_vlan_filter_set,
2813         .vlan_offload_set       = fm10k_vlan_offload_set,
2814         .mac_addr_add           = fm10k_macaddr_add,
2815         .mac_addr_remove        = fm10k_macaddr_remove,
2816         .rx_queue_start         = fm10k_dev_rx_queue_start,
2817         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2818         .tx_queue_start         = fm10k_dev_tx_queue_start,
2819         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2820         .rx_queue_setup         = fm10k_rx_queue_setup,
2821         .rx_queue_release       = fm10k_rx_queue_release,
2822         .tx_queue_setup         = fm10k_tx_queue_setup,
2823         .tx_queue_release       = fm10k_tx_queue_release,
2824         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2825         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2826         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2827         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2828         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2829         .reta_update            = fm10k_reta_update,
2830         .reta_query             = fm10k_reta_query,
2831         .rss_hash_update        = fm10k_rss_hash_update,
2832         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2833 };
2834
2835 static int ftag_check_handler(__rte_unused const char *key,
2836                 const char *value, __rte_unused void *opaque)
2837 {
2838         if (strcmp(value, "1"))
2839                 return -1;
2840
2841         return 0;
2842 }
2843
2844 static int
2845 fm10k_check_ftag(struct rte_devargs *devargs)
2846 {
2847         struct rte_kvargs *kvlist;
2848         const char *ftag_key = "enable_ftag";
2849
2850         if (devargs == NULL)
2851                 return 0;
2852
2853         kvlist = rte_kvargs_parse(devargs->args, NULL);
2854         if (kvlist == NULL)
2855                 return 0;
2856
2857         if (!rte_kvargs_count(kvlist, ftag_key)) {
2858                 rte_kvargs_free(kvlist);
2859                 return 0;
2860         }
2861         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2862         if (rte_kvargs_process(kvlist, ftag_key,
2863                                 ftag_check_handler, NULL) < 0) {
2864                 rte_kvargs_free(kvlist);
2865                 return 0;
2866         }
2867         rte_kvargs_free(kvlist);
2868
2869         return 1;
2870 }
2871
2872 static uint16_t
2873 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2874                     uint16_t nb_pkts)
2875 {
2876         uint16_t nb_tx = 0;
2877         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2878
2879         while (nb_pkts) {
2880                 uint16_t ret, num;
2881
2882                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2883                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2884                                                  num);
2885                 nb_tx += ret;
2886                 nb_pkts -= ret;
2887                 if (ret < num)
2888                         break;
2889         }
2890
2891         return nb_tx;
2892 }
2893
2894 static void __attribute__((cold))
2895 fm10k_set_tx_function(struct rte_eth_dev *dev)
2896 {
2897         struct fm10k_tx_queue *txq;
2898         int i;
2899         int use_sse = 1;
2900         uint16_t tx_ftag_en = 0;
2901
2902         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2903                 /* primary process has set the ftag flag and offloads */
2904                 txq = dev->data->tx_queues[0];
2905                 if (fm10k_tx_vec_condition_check(txq)) {
2906                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2907                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2908                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2909                 } else {
2910                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2911                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2912                         dev->tx_pkt_prepare = NULL;
2913                 }
2914                 return;
2915         }
2916
2917         if (fm10k_check_ftag(dev->device->devargs))
2918                 tx_ftag_en = 1;
2919
2920         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2921                 txq = dev->data->tx_queues[i];
2922                 txq->tx_ftag_en = tx_ftag_en;
2923                 /* Check if Vector Tx is satisfied */
2924                 if (fm10k_tx_vec_condition_check(txq))
2925                         use_sse = 0;
2926         }
2927
2928         if (use_sse) {
2929                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2930                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2931                         txq = dev->data->tx_queues[i];
2932                         fm10k_txq_vec_setup(txq);
2933                 }
2934                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2935                 dev->tx_pkt_prepare = NULL;
2936         } else {
2937                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2938                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2939                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2940         }
2941 }
2942
2943 static void __attribute__((cold))
2944 fm10k_set_rx_function(struct rte_eth_dev *dev)
2945 {
2946         struct fm10k_dev_info *dev_info =
2947                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2948         uint16_t i, rx_using_sse;
2949         uint16_t rx_ftag_en = 0;
2950
2951         if (fm10k_check_ftag(dev->device->devargs))
2952                 rx_ftag_en = 1;
2953
2954         /* In order to allow Vector Rx there are a few configuration
2955          * conditions to be met.
2956          */
2957         if (!fm10k_rx_vec_condition_check(dev) &&
2958                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2959                 if (dev->data->scattered_rx)
2960                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2961                 else
2962                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2963         } else if (dev->data->scattered_rx)
2964                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2965         else
2966                 dev->rx_pkt_burst = fm10k_recv_pkts;
2967
2968         rx_using_sse =
2969                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2970                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2971
2972         if (rx_using_sse)
2973                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2974         else
2975                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2976
2977         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2978                 return;
2979
2980         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2981                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2982
2983                 rxq->rx_using_sse = rx_using_sse;
2984                 rxq->rx_ftag_en = rx_ftag_en;
2985         }
2986 }
2987
2988 static void
2989 fm10k_params_init(struct rte_eth_dev *dev)
2990 {
2991         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2992         struct fm10k_dev_info *info =
2993                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2994
2995         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2996          * there is no way to get link status without reading BAR4.  Until this
2997          * works, assume we have maximum bandwidth.
2998          * @todo - fix bus info
2999          */
3000         hw->bus_caps.speed = fm10k_bus_speed_8000;
3001         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3002         hw->bus_caps.payload = fm10k_bus_payload_512;
3003         hw->bus.speed = fm10k_bus_speed_8000;
3004         hw->bus.width = fm10k_bus_width_pcie_x8;
3005         hw->bus.payload = fm10k_bus_payload_256;
3006
3007         info->rx_vec_allowed = true;
3008 }
3009
3010 static int
3011 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3012 {
3013         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3014         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3015         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3016         int diag, i;
3017         struct fm10k_macvlan_filter_info *macvlan;
3018
3019         PMD_INIT_FUNC_TRACE();
3020
3021         dev->dev_ops = &fm10k_eth_dev_ops;
3022         dev->rx_pkt_burst = &fm10k_recv_pkts;
3023         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3024         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3025
3026         /*
3027          * Primary process does the whole initialization, for secondary
3028          * processes, we just select the same Rx and Tx function as primary.
3029          */
3030         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3031                 fm10k_set_rx_function(dev);
3032                 fm10k_set_tx_function(dev);
3033                 return 0;
3034         }
3035
3036         rte_eth_copy_pci_info(dev, pdev);
3037
3038         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3039         memset(macvlan, 0, sizeof(*macvlan));
3040         /* Vendor and Device ID need to be set before init of shared code */
3041         memset(hw, 0, sizeof(*hw));
3042         hw->device_id = pdev->id.device_id;
3043         hw->vendor_id = pdev->id.vendor_id;
3044         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3045         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3046         hw->revision_id = 0;
3047         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3048         if (hw->hw_addr == NULL) {
3049                 PMD_INIT_LOG(ERR, "Bad mem resource."
3050                         " Try to blacklist unused devices.");
3051                 return -EIO;
3052         }
3053
3054         /* Store fm10k_adapter pointer */
3055         hw->back = dev->data->dev_private;
3056
3057         /* Initialize the shared code */
3058         diag = fm10k_init_shared_code(hw);
3059         if (diag != FM10K_SUCCESS) {
3060                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3061                 return -EIO;
3062         }
3063
3064         /* Initialize parameters */
3065         fm10k_params_init(dev);
3066
3067         /* Initialize the hw */
3068         diag = fm10k_init_hw(hw);
3069         if (diag != FM10K_SUCCESS) {
3070                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3071                 return -EIO;
3072         }
3073
3074         /* Initialize MAC address(es) */
3075         dev->data->mac_addrs = rte_zmalloc("fm10k",
3076                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3077         if (dev->data->mac_addrs == NULL) {
3078                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3079                 return -ENOMEM;
3080         }
3081
3082         diag = fm10k_read_mac_addr(hw);
3083
3084         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3085                         &dev->data->mac_addrs[0]);
3086
3087         if (diag != FM10K_SUCCESS ||
3088                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3089
3090                 /* Generate a random addr */
3091                 eth_random_addr(hw->mac.addr);
3092                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3093                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3094                 &dev->data->mac_addrs[0]);
3095         }
3096
3097         /* Reset the hw statistics */
3098         fm10k_stats_reset(dev);
3099
3100         /* Reset the hw */
3101         diag = fm10k_reset_hw(hw);
3102         if (diag != FM10K_SUCCESS) {
3103                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3104                 return -EIO;
3105         }
3106
3107         /* Setup mailbox service */
3108         diag = fm10k_setup_mbx_service(hw);
3109         if (diag != FM10K_SUCCESS) {
3110                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3111                 return -EIO;
3112         }
3113
3114         /*PF/VF has different interrupt handling mechanism */
3115         if (hw->mac.type == fm10k_mac_pf) {
3116                 /* register callback func to eal lib */
3117                 rte_intr_callback_register(intr_handle,
3118                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3119
3120                 /* enable MISC interrupt */
3121                 fm10k_dev_enable_intr_pf(dev);
3122         } else { /* VF */
3123                 rte_intr_callback_register(intr_handle,
3124                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3125
3126                 fm10k_dev_enable_intr_vf(dev);
3127         }
3128
3129         /* Enable intr after callback registered */
3130         rte_intr_enable(intr_handle);
3131
3132         hw->mac.ops.update_int_moderator(hw);
3133
3134         /* Make sure Switch Manager is ready before going forward. */
3135         if (hw->mac.type == fm10k_mac_pf) {
3136                 int switch_ready = 0;
3137
3138                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3139                         fm10k_mbx_lock(hw);
3140                         hw->mac.ops.get_host_state(hw, &switch_ready);
3141                         fm10k_mbx_unlock(hw);
3142                         if (switch_ready)
3143                                 break;
3144                         /* Delay some time to acquire async LPORT_MAP info. */
3145                         rte_delay_us(WAIT_SWITCH_MSG_US);
3146                 }
3147
3148                 if (switch_ready == 0) {
3149                         PMD_INIT_LOG(ERR, "switch is not ready");
3150                         return -1;
3151                 }
3152         }
3153
3154         /*
3155          * Below function will trigger operations on mailbox, acquire lock to
3156          * avoid race condition from interrupt handler. Operations on mailbox
3157          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3158          * will handle and generate an interrupt to our side. Then,  FIFO in
3159          * mailbox will be touched.
3160          */
3161         fm10k_mbx_lock(hw);
3162         /* Enable port first */
3163         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3164                                         MAX_LPORT_NUM, 1);
3165
3166         /* Set unicast mode by default. App can change to other mode in other
3167          * API func.
3168          */
3169         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3170                                         FM10K_XCAST_MODE_NONE);
3171
3172         fm10k_mbx_unlock(hw);
3173
3174         /* Make sure default VID is ready before going forward. */
3175         if (hw->mac.type == fm10k_mac_pf) {
3176                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3177                         if (hw->mac.default_vid)
3178                                 break;
3179                         /* Delay some time to acquire async port VLAN info. */
3180                         rte_delay_us(WAIT_SWITCH_MSG_US);
3181                 }
3182
3183                 if (!hw->mac.default_vid) {
3184                         PMD_INIT_LOG(ERR, "default VID is not ready");
3185                         return -1;
3186                 }
3187         }
3188
3189         /* Add default mac address */
3190         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3191                 MAIN_VSI_POOL_NUMBER);
3192
3193         return 0;
3194 }
3195
3196 static int
3197 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3198 {
3199         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3200         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3201         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3202         PMD_INIT_FUNC_TRACE();
3203
3204         /* only uninitialize in the primary process */
3205         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3206                 return 0;
3207
3208         /* safe to close dev here */
3209         fm10k_dev_close(dev);
3210
3211         dev->dev_ops = NULL;
3212         dev->rx_pkt_burst = NULL;
3213         dev->tx_pkt_burst = NULL;
3214
3215         /* disable uio/vfio intr */
3216         rte_intr_disable(intr_handle);
3217
3218         /*PF/VF has different interrupt handling mechanism */
3219         if (hw->mac.type == fm10k_mac_pf) {
3220                 /* disable interrupt */
3221                 fm10k_dev_disable_intr_pf(dev);
3222
3223                 /* unregister callback func to eal lib */
3224                 rte_intr_callback_unregister(intr_handle,
3225                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3226         } else {
3227                 /* disable interrupt */
3228                 fm10k_dev_disable_intr_vf(dev);
3229
3230                 rte_intr_callback_unregister(intr_handle,
3231                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3232         }
3233
3234         /* free mac memory */
3235         if (dev->data->mac_addrs) {
3236                 rte_free(dev->data->mac_addrs);
3237                 dev->data->mac_addrs = NULL;
3238         }
3239
3240         memset(hw, 0, sizeof(*hw));
3241
3242         return 0;
3243 }
3244
3245 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3246         struct rte_pci_device *pci_dev)
3247 {
3248         return rte_eth_dev_pci_generic_probe(pci_dev,
3249                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3250 }
3251
3252 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3253 {
3254         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3255 }
3256
3257 /*
3258  * The set of PCI devices this driver supports. This driver will enable both PF
3259  * and SRIOV-VF devices.
3260  */
3261 static const struct rte_pci_id pci_id_fm10k_map[] = {
3262         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3263         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3264         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3265         { .vendor_id = 0, /* sentinel */ },
3266 };
3267
3268 static struct rte_pci_driver rte_pmd_fm10k = {
3269         .id_table = pci_id_fm10k_map,
3270         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3271                      RTE_PCI_DRV_IOVA_AS_VA,
3272         .probe = eth_fm10k_pci_probe,
3273         .remove = eth_fm10k_pci_remove,
3274 };
3275
3276 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3277 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3278 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3279
3280 RTE_INIT(fm10k_init_log)
3281 {
3282         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3283         if (fm10k_logtype_init >= 0)
3284                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3285         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3286         if (fm10k_logtype_driver >= 0)
3287                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3288 }