drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 __rte_weak int
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 __rte_weak uint16_t
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 __rte_weak uint16_t
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 __rte_weak int
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 __rte_weak void
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 __rte_weak void
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 __rte_weak int
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 __rte_weak uint16_t
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static int
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return 0;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS) {
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930                 return -EAGAIN;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int
 937 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 938 {
 939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         uint8_t mode;
 941         int status;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Return if it didn't acquire valid glort range */
 946         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 947                 return 0;
 948
 949         if (dev->data->all_multicast == 1)
 950                 mode = FM10K_XCAST_MODE_ALLMULTI;
 951         else
 952                 mode = FM10K_XCAST_MODE_NONE;
 953
 954         fm10k_mbx_lock(hw);
 955         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 956                                 mode);
 957         fm10k_mbx_unlock(hw);
 958
 959         if (status != FM10K_SUCCESS) {
 960                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 961                 return -EAGAIN;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static void
 968 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971         int status;
 972
 973         PMD_INIT_FUNC_TRACE();
 974
 975         /* Return if it didn't acquire valid glort range */
 976         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 977                 return;
 978
 979         /* If promiscuous mode is enabled, it doesn't make sense to enable
 980          * allmulticast and disable promiscuous since fm10k only can select
 981          * one of the modes.
 982          */
 983         if (dev->data->promiscuous) {
 984                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 985                         "needn't enable allmulticast");
 986                 return;
 987         }
 988
 989         fm10k_mbx_lock(hw);
 990         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 991                                 FM10K_XCAST_MODE_ALLMULTI);
 992         fm10k_mbx_unlock(hw);
 993
 994         if (status != FM10K_SUCCESS)
 995                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 996 }
 997
 998 static void
 999 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1000 {
1001         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1002         int status;
1003
1004         PMD_INIT_FUNC_TRACE();
1005
1006         /* Return if it didn't acquire valid glort range */
1007         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1008                 return;
1009
1010         if (dev->data->promiscuous) {
1011                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1012                         "since promisc mode is enabled");
1013                 return;
1014         }
1015
1016         fm10k_mbx_lock(hw);
1017         /* Change mode to unicast mode */
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_NONE);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1031         uint16_t nb_queue_pools;
1032         struct fm10k_macvlan_filter_info *macvlan;
1033
1034         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1035         nb_queue_pools = macvlan->nb_queue_pools;
1036         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1037         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1038
1039         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1040         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1041         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1042                         hw->mac.dglort_map;
1043         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1044         /* Configure VMDQ/RSS DGlort Decoder */
1045         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1046
1047         /* Flow Director configurations, only queue number is valid. */
1048         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1049         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1050                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1051         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1052         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1053
1054         /* Invalidate all other GLORT entries */
1055         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1056                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1057                                 FM10K_DGLORTMAP_NONE);
1058 }
1059
1060 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1061 static int
1062 fm10k_dev_start(struct rte_eth_dev *dev)
1063 {
1064         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1065         int i, diag;
1066
1067         PMD_INIT_FUNC_TRACE();
1068
1069         /* stop, init, then start the hw */
1070         diag = fm10k_stop_hw(hw);
1071         if (diag != FM10K_SUCCESS) {
1072                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1073                 return -EIO;
1074         }
1075
1076         diag = fm10k_init_hw(hw);
1077         if (diag != FM10K_SUCCESS) {
1078                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1079                 return -EIO;
1080         }
1081
1082         diag = fm10k_start_hw(hw);
1083         if (diag != FM10K_SUCCESS) {
1084                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1085                 return -EIO;
1086         }
1087
1088         diag = fm10k_dev_tx_init(dev);
1089         if (diag) {
1090                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1091                 return diag;
1092         }
1093
1094         if (fm10k_dev_rxq_interrupt_setup(dev))
1095                 return -EIO;
1096
1097         diag = fm10k_dev_rx_init(dev);
1098         if (diag) {
1099                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1100                 return diag;
1101         }
1102
1103         if (hw->mac.type == fm10k_mac_pf)
1104                 fm10k_dev_dglort_map_configure(dev);
1105
1106         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1107                 struct fm10k_rx_queue *rxq;
1108                 rxq = dev->data->rx_queues[i];
1109
1110                 if (rxq->rx_deferred_start)
1111                         continue;
1112                 diag = fm10k_dev_rx_queue_start(dev, i);
1113                 if (diag != 0) {
1114                         int j;
1115                         for (j = 0; j < i; ++j)
1116                                 rx_queue_clean(dev->data->rx_queues[j]);
1117                         return diag;
1118                 }
1119         }
1120
1121         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1122                 struct fm10k_tx_queue *txq;
1123                 txq = dev->data->tx_queues[i];
1124
1125                 if (txq->tx_deferred_start)
1126                         continue;
1127                 diag = fm10k_dev_tx_queue_start(dev, i);
1128                 if (diag != 0) {
1129                         int j;
1130                         for (j = 0; j < i; ++j)
1131                                 tx_queue_clean(dev->data->tx_queues[j]);
1132                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1133                                 rx_queue_clean(dev->data->rx_queues[j]);
1134                         return diag;
1135                 }
1136         }
1137
1138         /* Update default vlan when not in VMDQ mode */
1139         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1140                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1141
1142         fm10k_link_update(dev, 0);
1143
1144         return 0;
1145 }
1146
1147 static void
1148 fm10k_dev_stop(struct rte_eth_dev *dev)
1149 {
1150         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1151         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1152         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1153         int i;
1154
1155         PMD_INIT_FUNC_TRACE();
1156
1157         if (dev->data->tx_queues)
1158                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1159                         fm10k_dev_tx_queue_stop(dev, i);
1160
1161         if (dev->data->rx_queues)
1162                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1163                         fm10k_dev_rx_queue_stop(dev, i);
1164
1165         /* Disable datapath event */
1166         if (rte_intr_dp_is_en(intr_handle)) {
1167                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1168                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1169                                 3 << FM10K_RXINT_TIMER_SHIFT);
1170                         if (hw->mac.type == fm10k_mac_pf)
1171                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1172                                         FM10K_ITR_MASK_SET);
1173                         else
1174                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1175                                         FM10K_ITR_MASK_SET);
1176                 }
1177         }
1178         /* Clean datapath event and queue/vec mapping */
1179         rte_intr_efd_disable(intr_handle);
1180         rte_free(intr_handle->intr_vec);
1181         intr_handle->intr_vec = NULL;
1182 }
1183
1184 static void
1185 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1186 {
1187         int i;
1188
1189         PMD_INIT_FUNC_TRACE();
1190
1191         if (dev->data->tx_queues) {
1192                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1193                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1194
1195                         tx_queue_free(txq);
1196                 }
1197         }
1198
1199         if (dev->data->rx_queues) {
1200                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1201                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1202         }
1203 }
1204
1205 static void
1206 fm10k_dev_close(struct rte_eth_dev *dev)
1207 {
1208         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1209
1210         PMD_INIT_FUNC_TRACE();
1211
1212         fm10k_mbx_lock(hw);
1213         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1214                 MAX_LPORT_NUM, false);
1215         fm10k_mbx_unlock(hw);
1216
1217         /* allow 100ms for device to quiesce */
1218         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1219
1220         /* Stop mailbox service first */
1221         fm10k_close_mbx_service(hw);
1222         fm10k_dev_stop(dev);
1223         fm10k_dev_queue_release(dev);
1224         fm10k_stop_hw(hw);
1225 }
1226
1227 static int
1228 fm10k_link_update(struct rte_eth_dev *dev,
1229         __rte_unused int wait_to_complete)
1230 {
1231         struct fm10k_dev_info *dev_info =
1232                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1233         PMD_INIT_FUNC_TRACE();
1234
1235         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1236         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1237         dev->data->dev_link.link_status =
1238                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1239         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1240
1241         return 0;
1242 }
1243
1244 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1245         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1246 {
1247         unsigned i, q;
1248         unsigned count = 0;
1249
1250         if (xstats_names != NULL) {
1251                 /* Note: limit checked in rte_eth_xstats_names() */
1252
1253                 /* Global stats */
1254                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1255                         snprintf(xstats_names[count].name,
1256                                 sizeof(xstats_names[count].name),
1257                                 "%s", fm10k_hw_stats_strings[count].name);
1258                         count++;
1259                 }
1260
1261                 /* PF queue stats */
1262                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1263                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1264                                 snprintf(xstats_names[count].name,
1265                                         sizeof(xstats_names[count].name),
1266                                         "rx_q%u_%s", q,
1267                                         fm10k_hw_stats_rx_q_strings[i].name);
1268                                 count++;
1269                         }
1270                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1271                                 snprintf(xstats_names[count].name,
1272                                         sizeof(xstats_names[count].name),
1273                                         "tx_q%u_%s", q,
1274                                         fm10k_hw_stats_tx_q_strings[i].name);
1275                                 count++;
1276                         }
1277                 }
1278         }
1279         return FM10K_NB_XSTATS;
1280 }
1281
1282 static int
1283 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1284                  unsigned n)
1285 {
1286         struct fm10k_hw_stats *hw_stats =
1287                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1288         unsigned i, q, count = 0;
1289
1290         if (n < FM10K_NB_XSTATS)
1291                 return FM10K_NB_XSTATS;
1292
1293         /* Global stats */
1294         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1295                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1296                         fm10k_hw_stats_strings[count].offset);
1297                 xstats[count].id = count;
1298                 count++;
1299         }
1300
1301         /* PF queue stats */
1302         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1303                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1304                         xstats[count].value =
1305                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1306                                 fm10k_hw_stats_rx_q_strings[i].offset);
1307                         xstats[count].id = count;
1308                         count++;
1309                 }
1310                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1311                         xstats[count].value =
1312                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1313                                 fm10k_hw_stats_tx_q_strings[i].offset);
1314                         xstats[count].id = count;
1315                         count++;
1316                 }
1317         }
1318
1319         return FM10K_NB_XSTATS;
1320 }
1321
1322 static int
1323 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1324 {
1325         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1326         struct fm10k_hw *hw =
1327                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1328         struct fm10k_hw_stats *hw_stats =
1329                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1330         int i;
1331
1332         PMD_INIT_FUNC_TRACE();
1333
1334         fm10k_update_hw_stats(hw, hw_stats);
1335
1336         ipackets = opackets = ibytes = obytes = imissed = 0;
1337         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1338                 (i < hw->mac.max_queues); ++i) {
1339                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1340                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1341                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1342                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1343                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1344                 ipackets += stats->q_ipackets[i];
1345                 opackets += stats->q_opackets[i];
1346                 ibytes   += stats->q_ibytes[i];
1347                 obytes   += stats->q_obytes[i];
1348                 imissed  += stats->q_errors[i];
1349         }
1350         stats->ipackets = ipackets;
1351         stats->opackets = opackets;
1352         stats->ibytes = ibytes;
1353         stats->obytes = obytes;
1354         stats->imissed = imissed;
1355         return 0;
1356 }
1357
1358 static void
1359 fm10k_stats_reset(struct rte_eth_dev *dev)
1360 {
1361         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1362         struct fm10k_hw_stats *hw_stats =
1363                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1364
1365         PMD_INIT_FUNC_TRACE();
1366
1367         memset(hw_stats, 0, sizeof(*hw_stats));
1368         fm10k_rebind_hw_stats(hw, hw_stats);
1369 }
1370
1371 static int
1372 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1373         struct rte_eth_dev_info *dev_info)
1374 {
1375         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1376         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1377
1378         PMD_INIT_FUNC_TRACE();
1379
1380         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1381         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1382         dev_info->max_rx_queues      = hw->mac.max_queues;
1383         dev_info->max_tx_queues      = hw->mac.max_queues;
1384         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1385         dev_info->max_hash_mac_addrs = 0;
1386         dev_info->max_vfs            = pdev->max_vfs;
1387         dev_info->vmdq_pool_base     = 0;
1388         dev_info->vmdq_queue_base    = 0;
1389         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1390         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1391         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1392         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1393                                     dev_info->rx_queue_offload_capa;
1394         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1395         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1396                                     dev_info->tx_queue_offload_capa;
1397
1398         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1399         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1400         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1401                                         ETH_RSS_IPV6 |
1402                                         ETH_RSS_IPV6_EX |
1403                                         ETH_RSS_NONFRAG_IPV4_TCP |
1404                                         ETH_RSS_NONFRAG_IPV6_TCP |
1405                                         ETH_RSS_IPV6_TCP_EX |
1406                                         ETH_RSS_NONFRAG_IPV4_UDP |
1407                                         ETH_RSS_NONFRAG_IPV6_UDP |
1408                                         ETH_RSS_IPV6_UDP_EX;
1409
1410         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1411                 .rx_thresh = {
1412                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1413                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1414                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1415                 },
1416                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1417                 .rx_drop_en = 0,
1418                 .offloads = 0,
1419         };
1420
1421         dev_info->default_txconf = (struct rte_eth_txconf) {
1422                 .tx_thresh = {
1423                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1424                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1425                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1426                 },
1427                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1428                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1429                 .offloads = 0,
1430         };
1431
1432         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1433                 .nb_max = FM10K_MAX_RX_DESC,
1434                 .nb_min = FM10K_MIN_RX_DESC,
1435                 .nb_align = FM10K_MULT_RX_DESC,
1436         };
1437
1438         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1439                 .nb_max = FM10K_MAX_TX_DESC,
1440                 .nb_min = FM10K_MIN_TX_DESC,
1441                 .nb_align = FM10K_MULT_TX_DESC,
1442                 .nb_seg_max = FM10K_TX_MAX_SEG,
1443                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1444         };
1445
1446         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1447                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1448                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1449
1450         return 0;
1451 }
1452
1453 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1454 static const uint32_t *
1455 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1456 {
1457         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1458             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1459                 static uint32_t ptypes[] = {
1460                         /* refers to rx_desc_to_ol_flags() */
1461                         RTE_PTYPE_L2_ETHER,
1462                         RTE_PTYPE_L3_IPV4,
1463                         RTE_PTYPE_L3_IPV4_EXT,
1464                         RTE_PTYPE_L3_IPV6,
1465                         RTE_PTYPE_L3_IPV6_EXT,
1466                         RTE_PTYPE_L4_TCP,
1467                         RTE_PTYPE_L4_UDP,
1468                         RTE_PTYPE_UNKNOWN
1469                 };
1470
1471                 return ptypes;
1472         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1473                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1474                 static uint32_t ptypes_vec[] = {
1475                         /* refers to fm10k_desc_to_pktype_v() */
1476                         RTE_PTYPE_L3_IPV4,
1477                         RTE_PTYPE_L3_IPV4_EXT,
1478                         RTE_PTYPE_L3_IPV6,
1479                         RTE_PTYPE_L3_IPV6_EXT,
1480                         RTE_PTYPE_L4_TCP,
1481                         RTE_PTYPE_L4_UDP,
1482                         RTE_PTYPE_TUNNEL_GENEVE,
1483                         RTE_PTYPE_TUNNEL_NVGRE,
1484                         RTE_PTYPE_TUNNEL_VXLAN,
1485                         RTE_PTYPE_TUNNEL_GRE,
1486                         RTE_PTYPE_UNKNOWN
1487                 };
1488
1489                 return ptypes_vec;
1490         }
1491
1492         return NULL;
1493 }
1494 #else
1495 static const uint32_t *
1496 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1497 {
1498         return NULL;
1499 }
1500 #endif
1501
1502 static int
1503 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1504 {
1505         s32 result;
1506         uint16_t mac_num = 0;
1507         uint32_t vid_idx, vid_bit, mac_index;
1508         struct fm10k_hw *hw;
1509         struct fm10k_macvlan_filter_info *macvlan;
1510         struct rte_eth_dev_data *data = dev->data;
1511
1512         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1513         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1514
1515         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1516                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1517                 return -EINVAL;
1518         }
1519
1520         if (vlan_id > ETH_VLAN_ID_MAX) {
1521                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1522                 return -EINVAL;
1523         }
1524
1525         vid_idx = FM10K_VFTA_IDX(vlan_id);
1526         vid_bit = FM10K_VFTA_BIT(vlan_id);
1527         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1528         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1529                 return 0;
1530         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1531         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1532                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1533                         "in the VLAN filter table");
1534                 return -EINVAL;
1535         }
1536
1537         fm10k_mbx_lock(hw);
1538         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1539         fm10k_mbx_unlock(hw);
1540         if (result != FM10K_SUCCESS) {
1541                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1542                 return -EIO;
1543         }
1544
1545         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1546                         (result == FM10K_SUCCESS); mac_index++) {
1547                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1548                         continue;
1549                 if (mac_num > macvlan->mac_num - 1) {
1550                         PMD_INIT_LOG(ERR, "MAC address number "
1551                                         "not match");
1552                         break;
1553                 }
1554                 fm10k_mbx_lock(hw);
1555                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1556                         data->mac_addrs[mac_index].addr_bytes,
1557                         vlan_id, on, 0);
1558                 fm10k_mbx_unlock(hw);
1559                 mac_num++;
1560         }
1561         if (result != FM10K_SUCCESS) {
1562                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1563                 return -EIO;
1564         }
1565
1566         if (on) {
1567                 macvlan->vlan_num++;
1568                 macvlan->vfta[vid_idx] |= vid_bit;
1569         } else {
1570                 macvlan->vlan_num--;
1571                 macvlan->vfta[vid_idx] &= ~vid_bit;
1572         }
1573         return 0;
1574 }
1575
1576 static int
1577 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1578 {
1579         if (mask & ETH_VLAN_STRIP_MASK) {
1580                 if (!(dev->data->dev_conf.rxmode.offloads &
1581                         DEV_RX_OFFLOAD_VLAN_STRIP))
1582                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1583                                         "always on in fm10k");
1584         }
1585
1586         if (mask & ETH_VLAN_EXTEND_MASK) {
1587                 if (dev->data->dev_conf.rxmode.offloads &
1588                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1589                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1590                                         "supported in fm10k");
1591         }
1592
1593         if (mask & ETH_VLAN_FILTER_MASK) {
1594                 if (!(dev->data->dev_conf.rxmode.offloads &
1595                         DEV_RX_OFFLOAD_VLAN_FILTER))
1596                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1597         }
1598
1599         return 0;
1600 }
1601
1602 /* Add/Remove a MAC address, and update filters to main VSI */
1603 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1604                 const u8 *mac, bool add, uint32_t pool)
1605 {
1606         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1607         struct fm10k_macvlan_filter_info *macvlan;
1608         uint32_t i, j, k;
1609
1610         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1611
1612         if (pool != MAIN_VSI_POOL_NUMBER) {
1613                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1614                         "mac to pool %u", pool);
1615                 return;
1616         }
1617         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1618                 if (!macvlan->vfta[j])
1619                         continue;
1620                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1621                         if (!(macvlan->vfta[j] & (1 << k)))
1622                                 continue;
1623                         if (i + 1 > macvlan->vlan_num) {
1624                                 PMD_INIT_LOG(ERR, "vlan number not match");
1625                                 return;
1626                         }
1627                         fm10k_mbx_lock(hw);
1628                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1629                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1630                         fm10k_mbx_unlock(hw);
1631                         i++;
1632                 }
1633         }
1634 }
1635
1636 /* Add/Remove a MAC address, and update filters to VMDQ */
1637 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1638                 const u8 *mac, bool add, uint32_t pool)
1639 {
1640         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1641         struct fm10k_macvlan_filter_info *macvlan;
1642         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1643         uint32_t i;
1644
1645         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1646         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1647
1648         if (pool > macvlan->nb_queue_pools) {
1649                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1650                         " Max pool is %u",
1651                         pool, macvlan->nb_queue_pools);
1652                 return;
1653         }
1654         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1655                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1656                         continue;
1657                 fm10k_mbx_lock(hw);
1658                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1659                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1660                 fm10k_mbx_unlock(hw);
1661         }
1662 }
1663
1664 /* Add/Remove a MAC address, and update filters */
1665 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1666                 const u8 *mac, bool add, uint32_t pool)
1667 {
1668         struct fm10k_macvlan_filter_info *macvlan;
1669
1670         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1671
1672         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1673                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1674         else
1675                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1676
1677         if (add)
1678                 macvlan->mac_num++;
1679         else
1680                 macvlan->mac_num--;
1681 }
1682
1683 /* Add a MAC address, and update filters */
1684 static int
1685 fm10k_macaddr_add(struct rte_eth_dev *dev,
1686                 struct rte_ether_addr *mac_addr,
1687                 uint32_t index,
1688                 uint32_t pool)
1689 {
1690         struct fm10k_macvlan_filter_info *macvlan;
1691
1692         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1693         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1694         macvlan->mac_vmdq_id[index] = pool;
1695         return 0;
1696 }
1697
1698 /* Remove a MAC address, and update filters */
1699 static void
1700 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1701 {
1702         struct rte_eth_dev_data *data = dev->data;
1703         struct fm10k_macvlan_filter_info *macvlan;
1704
1705         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1706         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1707                         FALSE, macvlan->mac_vmdq_id[index]);
1708         macvlan->mac_vmdq_id[index] = 0;
1709 }
1710
1711 static inline int
1712 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1713 {
1714         if ((request < min) || (request > max) || ((request % mult) != 0))
1715                 return -1;
1716         else
1717                 return 0;
1718 }
1719
1720
1721 static inline int
1722 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1723 {
1724         if ((request < min) || (request > max) || ((div % request) != 0))
1725                 return -1;
1726         else
1727                 return 0;
1728 }
1729
1730 static inline int
1731 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1732 {
1733         uint16_t rx_free_thresh;
1734
1735         if (conf->rx_free_thresh == 0)
1736                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1737         else
1738                 rx_free_thresh = conf->rx_free_thresh;
1739
1740         /* make sure the requested threshold satisfies the constraints */
1741         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1742                         FM10K_RX_FREE_THRESH_MAX(q),
1743                         FM10K_RX_FREE_THRESH_DIV(q),
1744                         rx_free_thresh)) {
1745                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1746                         "less than or equal to %u, "
1747                         "greater than or equal to %u, "
1748                         "and a divisor of %u",
1749                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1750                         FM10K_RX_FREE_THRESH_MIN(q),
1751                         FM10K_RX_FREE_THRESH_DIV(q));
1752                 return -EINVAL;
1753         }
1754
1755         q->alloc_thresh = rx_free_thresh;
1756         q->drop_en = conf->rx_drop_en;
1757         q->rx_deferred_start = conf->rx_deferred_start;
1758
1759         return 0;
1760 }
1761
1762 /*
1763  * Hardware requires specific alignment for Rx packet buffers. At
1764  * least one of the following two conditions must be satisfied.
1765  *  1. Address is 512B aligned
1766  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1767  *
1768  * As such, the driver may need to adjust the DMA address within the
1769  * buffer by up to 512B.
1770  *
1771  * return 1 if the element size is valid, otherwise return 0.
1772  */
1773 static int
1774 mempool_element_size_valid(struct rte_mempool *mp)
1775 {
1776         uint32_t min_size;
1777
1778         /* elt_size includes mbuf header and headroom */
1779         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1780                         RTE_PKTMBUF_HEADROOM;
1781
1782         /* account for up to 512B of alignment */
1783         min_size -= FM10K_RX_DATABUF_ALIGN;
1784
1785         /* sanity check for overflow */
1786         if (min_size > mp->elt_size)
1787                 return 0;
1788
1789         /* size is valid */
1790         return 1;
1791 }
1792
1793 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1794 {
1795         RTE_SET_USED(dev);
1796
1797         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1798 }
1799
1800 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1801 {
1802         RTE_SET_USED(dev);
1803
1804         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1805                            DEV_RX_OFFLOAD_VLAN_FILTER |
1806                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1807                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1808                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1809                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1810                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1811 }
1812
1813 static int
1814 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1815         uint16_t nb_desc, unsigned int socket_id,
1816         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1817 {
1818         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1819         struct fm10k_dev_info *dev_info =
1820                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1821         struct fm10k_rx_queue *q;
1822         const struct rte_memzone *mz;
1823         uint64_t offloads;
1824
1825         PMD_INIT_FUNC_TRACE();
1826
1827         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1828
1829         /* make sure the mempool element size can account for alignment. */
1830         if (!mempool_element_size_valid(mp)) {
1831                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1832                 return -EINVAL;
1833         }
1834
1835         /* make sure a valid number of descriptors have been requested */
1836         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1837                                 FM10K_MULT_RX_DESC, nb_desc)) {
1838                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1839                         "less than or equal to %"PRIu32", "
1840                         "greater than or equal to %u, "
1841                         "and a multiple of %u",
1842                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1843                         FM10K_MULT_RX_DESC);
1844                 return -EINVAL;
1845         }
1846
1847         /*
1848          * if this queue existed already, free the associated memory. The
1849          * queue cannot be reused in case we need to allocate memory on
1850          * different socket than was previously used.
1851          */
1852         if (dev->data->rx_queues[queue_id] != NULL) {
1853                 rx_queue_free(dev->data->rx_queues[queue_id]);
1854                 dev->data->rx_queues[queue_id] = NULL;
1855         }
1856
1857         /* allocate memory for the queue structure */
1858         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1859                                 socket_id);
1860         if (q == NULL) {
1861                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1862                 return -ENOMEM;
1863         }
1864
1865         /* setup queue */
1866         q->mp = mp;
1867         q->nb_desc = nb_desc;
1868         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1869         q->port_id = dev->data->port_id;
1870         q->queue_id = queue_id;
1871         q->tail_ptr = (volatile uint32_t *)
1872                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1873         q->offloads = offloads;
1874         if (handle_rxconf(q, conf))
1875                 return -EINVAL;
1876
1877         /* allocate memory for the software ring */
1878         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1879                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1880                         RTE_CACHE_LINE_SIZE, socket_id);
1881         if (q->sw_ring == NULL) {
1882                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1883                 rte_free(q);
1884                 return -ENOMEM;
1885         }
1886
1887         /*
1888          * allocate memory for the hardware descriptor ring. A memzone large
1889          * enough to hold the maximum ring size is requested to allow for
1890          * resizing in later calls to the queue setup function.
1891          */
1892         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1893                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1894                                       socket_id);
1895         if (mz == NULL) {
1896                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1897                 rte_free(q->sw_ring);
1898                 rte_free(q);
1899                 return -ENOMEM;
1900         }
1901         q->hw_ring = mz->addr;
1902         q->hw_ring_phys_addr = mz->iova;
1903
1904         /* Check if number of descs satisfied Vector requirement */
1905         if (!rte_is_power_of_2(nb_desc)) {
1906                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1907                                     "preconditions - canceling the feature for "
1908                                     "the whole port[%d]",
1909                              q->queue_id, q->port_id);
1910                 dev_info->rx_vec_allowed = false;
1911         } else
1912                 fm10k_rxq_vec_setup(q);
1913
1914         dev->data->rx_queues[queue_id] = q;
1915         return 0;
1916 }
1917
1918 static void
1919 fm10k_rx_queue_release(void *queue)
1920 {
1921         PMD_INIT_FUNC_TRACE();
1922
1923         rx_queue_free(queue);
1924 }
1925
1926 static inline int
1927 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1928 {
1929         uint16_t tx_free_thresh;
1930         uint16_t tx_rs_thresh;
1931
1932         /* constraint MACROs require that tx_free_thresh is configured
1933          * before tx_rs_thresh */
1934         if (conf->tx_free_thresh == 0)
1935                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1936         else
1937                 tx_free_thresh = conf->tx_free_thresh;
1938
1939         /* make sure the requested threshold satisfies the constraints */
1940         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1941                         FM10K_TX_FREE_THRESH_MAX(q),
1942                         FM10K_TX_FREE_THRESH_DIV(q),
1943                         tx_free_thresh)) {
1944                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1945                         "less than or equal to %u, "
1946                         "greater than or equal to %u, "
1947                         "and a divisor of %u",
1948                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1949                         FM10K_TX_FREE_THRESH_MIN(q),
1950                         FM10K_TX_FREE_THRESH_DIV(q));
1951                 return -EINVAL;
1952         }
1953
1954         q->free_thresh = tx_free_thresh;
1955
1956         if (conf->tx_rs_thresh == 0)
1957                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1958         else
1959                 tx_rs_thresh = conf->tx_rs_thresh;
1960
1961         q->tx_deferred_start = conf->tx_deferred_start;
1962
1963         /* make sure the requested threshold satisfies the constraints */
1964         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1965                         FM10K_TX_RS_THRESH_MAX(q),
1966                         FM10K_TX_RS_THRESH_DIV(q),
1967                         tx_rs_thresh)) {
1968                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1969                         "less than or equal to %u, "
1970                         "greater than or equal to %u, "
1971                         "and a divisor of %u",
1972                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1973                         FM10K_TX_RS_THRESH_MIN(q),
1974                         FM10K_TX_RS_THRESH_DIV(q));
1975                 return -EINVAL;
1976         }
1977
1978         q->rs_thresh = tx_rs_thresh;
1979
1980         return 0;
1981 }
1982
1983 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1984 {
1985         RTE_SET_USED(dev);
1986
1987         return 0;
1988 }
1989
1990 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1991 {
1992         RTE_SET_USED(dev);
1993
1994         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1995                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1996                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1997                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1998                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1999                           DEV_TX_OFFLOAD_TCP_TSO);
2000 }
2001
2002 static int
2003 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2004         uint16_t nb_desc, unsigned int socket_id,
2005         const struct rte_eth_txconf *conf)
2006 {
2007         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2008         struct fm10k_tx_queue *q;
2009         const struct rte_memzone *mz;
2010         uint64_t offloads;
2011
2012         PMD_INIT_FUNC_TRACE();
2013
2014         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2015
2016         /* make sure a valid number of descriptors have been requested */
2017         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2018                                 FM10K_MULT_TX_DESC, nb_desc)) {
2019                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2020                         "less than or equal to %"PRIu32", "
2021                         "greater than or equal to %u, "
2022                         "and a multiple of %u",
2023                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2024                         FM10K_MULT_TX_DESC);
2025                 return -EINVAL;
2026         }
2027
2028         /*
2029          * if this queue existed already, free the associated memory. The
2030          * queue cannot be reused in case we need to allocate memory on
2031          * different socket than was previously used.
2032          */
2033         if (dev->data->tx_queues[queue_id] != NULL) {
2034                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2035
2036                 tx_queue_free(txq);
2037                 dev->data->tx_queues[queue_id] = NULL;
2038         }
2039
2040         /* allocate memory for the queue structure */
2041         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2042                                 socket_id);
2043         if (q == NULL) {
2044                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2045                 return -ENOMEM;
2046         }
2047
2048         /* setup queue */
2049         q->nb_desc = nb_desc;
2050         q->port_id = dev->data->port_id;
2051         q->queue_id = queue_id;
2052         q->offloads = offloads;
2053         q->ops = &def_txq_ops;
2054         q->tail_ptr = (volatile uint32_t *)
2055                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2056         if (handle_txconf(q, conf))
2057                 return -EINVAL;
2058
2059         /* allocate memory for the software ring */
2060         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2061                                         nb_desc * sizeof(struct rte_mbuf *),
2062                                         RTE_CACHE_LINE_SIZE, socket_id);
2063         if (q->sw_ring == NULL) {
2064                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2065                 rte_free(q);
2066                 return -ENOMEM;
2067         }
2068
2069         /*
2070          * allocate memory for the hardware descriptor ring. A memzone large
2071          * enough to hold the maximum ring size is requested to allow for
2072          * resizing in later calls to the queue setup function.
2073          */
2074         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2075                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2076                                       socket_id);
2077         if (mz == NULL) {
2078                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2079                 rte_free(q->sw_ring);
2080                 rte_free(q);
2081                 return -ENOMEM;
2082         }
2083         q->hw_ring = mz->addr;
2084         q->hw_ring_phys_addr = mz->iova;
2085
2086         /*
2087          * allocate memory for the RS bit tracker. Enough slots to hold the
2088          * descriptor index for each RS bit needing to be set are required.
2089          */
2090         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2091                                 ((nb_desc + 1) / q->rs_thresh) *
2092                                 sizeof(uint16_t),
2093                                 RTE_CACHE_LINE_SIZE, socket_id);
2094         if (q->rs_tracker.list == NULL) {
2095                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2096                 rte_free(q->sw_ring);
2097                 rte_free(q);
2098                 return -ENOMEM;
2099         }
2100
2101         dev->data->tx_queues[queue_id] = q;
2102         return 0;
2103 }
2104
2105 static void
2106 fm10k_tx_queue_release(void *queue)
2107 {
2108         struct fm10k_tx_queue *q = queue;
2109         PMD_INIT_FUNC_TRACE();
2110
2111         tx_queue_free(q);
2112 }
2113
2114 static int
2115 fm10k_reta_update(struct rte_eth_dev *dev,
2116                         struct rte_eth_rss_reta_entry64 *reta_conf,
2117                         uint16_t reta_size)
2118 {
2119         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2120         uint16_t i, j, idx, shift;
2121         uint8_t mask;
2122         uint32_t reta;
2123
2124         PMD_INIT_FUNC_TRACE();
2125
2126         if (reta_size > FM10K_MAX_RSS_INDICES) {
2127                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2128                         "(%d) doesn't match the number hardware can supported "
2129                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2130                 return -EINVAL;
2131         }
2132
2133         /*
2134          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2135          * 128-entries in 32 registers
2136          */
2137         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2138                 idx = i / RTE_RETA_GROUP_SIZE;
2139                 shift = i % RTE_RETA_GROUP_SIZE;
2140                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2141                                 BIT_MASK_PER_UINT32);
2142                 if (mask == 0)
2143                         continue;
2144
2145                 reta = 0;
2146                 if (mask != BIT_MASK_PER_UINT32)
2147                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2148
2149                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2150                         if (mask & (0x1 << j)) {
2151                                 if (mask != 0xF)
2152                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2153                                 reta |= reta_conf[idx].reta[shift + j] <<
2154                                                 (CHAR_BIT * j);
2155                         }
2156                 }
2157                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2158         }
2159
2160         return 0;
2161 }
2162
2163 static int
2164 fm10k_reta_query(struct rte_eth_dev *dev,
2165                         struct rte_eth_rss_reta_entry64 *reta_conf,
2166                         uint16_t reta_size)
2167 {
2168         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2169         uint16_t i, j, idx, shift;
2170         uint8_t mask;
2171         uint32_t reta;
2172
2173         PMD_INIT_FUNC_TRACE();
2174
2175         if (reta_size < FM10K_MAX_RSS_INDICES) {
2176                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2177                         "(%d) doesn't match the number hardware can supported "
2178                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2179                 return -EINVAL;
2180         }
2181
2182         /*
2183          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2184          * 128-entries in 32 registers
2185          */
2186         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2187                 idx = i / RTE_RETA_GROUP_SIZE;
2188                 shift = i % RTE_RETA_GROUP_SIZE;
2189                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2190                                 BIT_MASK_PER_UINT32);
2191                 if (mask == 0)
2192                         continue;
2193
2194                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2195                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2196                         if (mask & (0x1 << j))
2197                                 reta_conf[idx].reta[shift + j] = ((reta >>
2198                                         CHAR_BIT * j) & UINT8_MAX);
2199                 }
2200         }
2201
2202         return 0;
2203 }
2204
2205 static int
2206 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2207         struct rte_eth_rss_conf *rss_conf)
2208 {
2209         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2210         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2211         uint32_t mrqc;
2212         uint64_t hf = rss_conf->rss_hf;
2213         int i;
2214
2215         PMD_INIT_FUNC_TRACE();
2216
2217         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2218                                 FM10K_RSSRK_ENTRIES_PER_REG))
2219                 return -EINVAL;
2220
2221         if (hf == 0)
2222                 return -EINVAL;
2223
2224         mrqc = 0;
2225         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2226         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2227         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2228         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2229         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2230         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2231         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2232         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2233         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2234
2235         /* If the mapping doesn't fit any supported, return */
2236         if (mrqc == 0)
2237                 return -EINVAL;
2238
2239         if (key != NULL)
2240                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2241                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2242
2243         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2244
2245         return 0;
2246 }
2247
2248 static int
2249 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2250         struct rte_eth_rss_conf *rss_conf)
2251 {
2252         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2253         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2254         uint32_t mrqc;
2255         uint64_t hf;
2256         int i;
2257
2258         PMD_INIT_FUNC_TRACE();
2259
2260         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2261                                 FM10K_RSSRK_ENTRIES_PER_REG))
2262                 return -EINVAL;
2263
2264         if (key != NULL)
2265                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2266                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2267
2268         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2269         hf = 0;
2270         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2271         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2272         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2273         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2274         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2275         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2276         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2277         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2278         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2279
2280         rss_conf->rss_hf = hf;
2281
2282         return 0;
2283 }
2284
2285 static void
2286 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2287 {
2288         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2289         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2290
2291         /* Bind all local non-queue interrupt to vector 0 */
2292         int_map |= FM10K_MISC_VEC_ID;
2293
2294         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2298         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2299         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2300
2301         /* Enable misc causes */
2302         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2303                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2304                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2305                                 FM10K_EIMR_ENABLE(MAILBOX) |
2306                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2307                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2308                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2309                                 FM10K_EIMR_ENABLE(VFLR));
2310
2311         /* Enable ITR 0 */
2312         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2313                                         FM10K_ITR_MASK_CLEAR);
2314         FM10K_WRITE_FLUSH(hw);
2315 }
2316
2317 static void
2318 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2319 {
2320         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2321         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2322
2323         int_map |= FM10K_MISC_VEC_ID;
2324
2325         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2326         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2327         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2328         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2329         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2330         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2331
2332         /* Disable misc causes */
2333         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2334                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2335                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2336                                 FM10K_EIMR_DISABLE(MAILBOX) |
2337                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2338                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2339                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2340                                 FM10K_EIMR_DISABLE(VFLR));
2341
2342         /* Disable ITR 0 */
2343         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2344         FM10K_WRITE_FLUSH(hw);
2345 }
2346
2347 static void
2348 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2349 {
2350         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2351         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2352
2353         /* Bind all local non-queue interrupt to vector 0 */
2354         int_map |= FM10K_MISC_VEC_ID;
2355
2356         /* Only INT 0 available, other 15 are reserved. */
2357         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2358
2359         /* Enable ITR 0 */
2360         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2361                                         FM10K_ITR_MASK_CLEAR);
2362         FM10K_WRITE_FLUSH(hw);
2363 }
2364
2365 static void
2366 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2367 {
2368         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2369         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2370
2371         int_map |= FM10K_MISC_VEC_ID;
2372
2373         /* Only INT 0 available, other 15 are reserved. */
2374         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2375
2376         /* Disable ITR 0 */
2377         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2378         FM10K_WRITE_FLUSH(hw);
2379 }
2380
2381 static int
2382 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2383 {
2384         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2385         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2386
2387         /* Enable ITR */
2388         if (hw->mac.type == fm10k_mac_pf)
2389                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2390                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2391         else
2392                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2393                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2394         rte_intr_ack(&pdev->intr_handle);
2395         return 0;
2396 }
2397
2398 static int
2399 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2400 {
2401         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2402         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2403
2404         /* Disable ITR */
2405         if (hw->mac.type == fm10k_mac_pf)
2406                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2407                         FM10K_ITR_MASK_SET);
2408         else
2409                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2410                         FM10K_ITR_MASK_SET);
2411         return 0;
2412 }
2413
2414 static int
2415 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2416 {
2417         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2418         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2419         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2420         uint32_t intr_vector, vec;
2421         uint16_t queue_id;
2422         int result = 0;
2423
2424         /* fm10k needs one separate interrupt for mailbox,
2425          * so only drivers which support multiple interrupt vectors
2426          * e.g. vfio-pci can work for fm10k interrupt mode
2427          */
2428         if (!rte_intr_cap_multiple(intr_handle) ||
2429                         dev->data->dev_conf.intr_conf.rxq == 0)
2430                 return result;
2431
2432         intr_vector = dev->data->nb_rx_queues;
2433
2434         /* disable interrupt first */
2435         rte_intr_disable(intr_handle);
2436         if (hw->mac.type == fm10k_mac_pf)
2437                 fm10k_dev_disable_intr_pf(dev);
2438         else
2439                 fm10k_dev_disable_intr_vf(dev);
2440
2441         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2442                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2443                 result = -EIO;
2444         }
2445
2446         if (rte_intr_dp_is_en(intr_handle) && !result) {
2447                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2448                         dev->data->nb_rx_queues * sizeof(int), 0);
2449                 if (intr_handle->intr_vec) {
2450                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2451                                         queue_id < dev->data->nb_rx_queues;
2452                                         queue_id++) {
2453                                 intr_handle->intr_vec[queue_id] = vec;
2454                                 if (vec < intr_handle->nb_efd - 1
2455                                                 + FM10K_RX_VEC_START)
2456                                         vec++;
2457                         }
2458                 } else {
2459                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2460                                 " intr_vec", dev->data->nb_rx_queues);
2461                         rte_intr_efd_disable(intr_handle);
2462                         result = -ENOMEM;
2463                 }
2464         }
2465
2466         if (hw->mac.type == fm10k_mac_pf)
2467                 fm10k_dev_enable_intr_pf(dev);
2468         else
2469                 fm10k_dev_enable_intr_vf(dev);
2470         rte_intr_enable(intr_handle);
2471         hw->mac.ops.update_int_moderator(hw);
2472         return result;
2473 }
2474
2475 static int
2476 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2477 {
2478         struct fm10k_fault fault;
2479         int err;
2480         const char *estr = "Unknown error";
2481
2482         /* Process PCA fault */
2483         if (eicr & FM10K_EICR_PCA_FAULT) {
2484                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2485                 if (err)
2486                         goto error;
2487                 switch (fault.type) {
2488                 case PCA_NO_FAULT:
2489                         estr = "PCA_NO_FAULT"; break;
2490                 case PCA_UNMAPPED_ADDR:
2491                         estr = "PCA_UNMAPPED_ADDR"; break;
2492                 case PCA_BAD_QACCESS_PF:
2493                         estr = "PCA_BAD_QACCESS_PF"; break;
2494                 case PCA_BAD_QACCESS_VF:
2495                         estr = "PCA_BAD_QACCESS_VF"; break;
2496                 case PCA_MALICIOUS_REQ:
2497                         estr = "PCA_MALICIOUS_REQ"; break;
2498                 case PCA_POISONED_TLP:
2499                         estr = "PCA_POISONED_TLP"; break;
2500                 case PCA_TLP_ABORT:
2501                         estr = "PCA_TLP_ABORT"; break;
2502                 default:
2503                         goto error;
2504                 }
2505                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2506                         estr, fault.func ? "VF" : "PF", fault.func,
2507                         fault.address, fault.specinfo);
2508         }
2509
2510         /* Process THI fault */
2511         if (eicr & FM10K_EICR_THI_FAULT) {
2512                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2513                 if (err)
2514                         goto error;
2515                 switch (fault.type) {
2516                 case THI_NO_FAULT:
2517                         estr = "THI_NO_FAULT"; break;
2518                 case THI_MAL_DIS_Q_FAULT:
2519                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2520                 default:
2521                         goto error;
2522                 }
2523                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2524                         estr, fault.func ? "VF" : "PF", fault.func,
2525                         fault.address, fault.specinfo);
2526         }
2527
2528         /* Process FUM fault */
2529         if (eicr & FM10K_EICR_FUM_FAULT) {
2530                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2531                 if (err)
2532                         goto error;
2533                 switch (fault.type) {
2534                 case FUM_NO_FAULT:
2535                         estr = "FUM_NO_FAULT"; break;
2536                 case FUM_UNMAPPED_ADDR:
2537                         estr = "FUM_UNMAPPED_ADDR"; break;
2538                 case FUM_POISONED_TLP:
2539                         estr = "FUM_POISONED_TLP"; break;
2540                 case FUM_BAD_VF_QACCESS:
2541                         estr = "FUM_BAD_VF_QACCESS"; break;
2542                 case FUM_ADD_DECODE_ERR:
2543                         estr = "FUM_ADD_DECODE_ERR"; break;
2544                 case FUM_RO_ERROR:
2545                         estr = "FUM_RO_ERROR"; break;
2546                 case FUM_QPRC_CRC_ERROR:
2547                         estr = "FUM_QPRC_CRC_ERROR"; break;
2548                 case FUM_CSR_TIMEOUT:
2549                         estr = "FUM_CSR_TIMEOUT"; break;
2550                 case FUM_INVALID_TYPE:
2551                         estr = "FUM_INVALID_TYPE"; break;
2552                 case FUM_INVALID_LENGTH:
2553                         estr = "FUM_INVALID_LENGTH"; break;
2554                 case FUM_INVALID_BE:
2555                         estr = "FUM_INVALID_BE"; break;
2556                 case FUM_INVALID_ALIGN:
2557                         estr = "FUM_INVALID_ALIGN"; break;
2558                 default:
2559                         goto error;
2560                 }
2561                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2562                         estr, fault.func ? "VF" : "PF", fault.func,
2563                         fault.address, fault.specinfo);
2564         }
2565
2566         return 0;
2567 error:
2568         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2569         return err;
2570 }
2571
2572 /**
2573  * PF interrupt handler triggered by NIC for handling specific interrupt.
2574  *
2575  * @param handle
2576  *  Pointer to interrupt handle.
2577  * @param param
2578  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2579  *
2580  * @return
2581  *  void
2582  */
2583 static void
2584 fm10k_dev_interrupt_handler_pf(void *param)
2585 {
2586         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2587         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2588         uint32_t cause, status;
2589         struct fm10k_dev_info *dev_info =
2590                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2591         int status_mbx;
2592         s32 err;
2593
2594         if (hw->mac.type != fm10k_mac_pf)
2595                 return;
2596
2597         cause = FM10K_READ_REG(hw, FM10K_EICR);
2598
2599         /* Handle PCI fault cases */
2600         if (cause & FM10K_EICR_FAULT_MASK) {
2601                 PMD_INIT_LOG(ERR, "INT: find fault!");
2602                 fm10k_dev_handle_fault(hw, cause);
2603         }
2604
2605         /* Handle switch up/down */
2606         if (cause & FM10K_EICR_SWITCHNOTREADY)
2607                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2608
2609         if (cause & FM10K_EICR_SWITCHREADY) {
2610                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2611                 if (dev_info->sm_down == 1) {
2612                         fm10k_mbx_lock(hw);
2613
2614                         /* For recreating logical ports */
2615                         status_mbx = hw->mac.ops.update_lport_state(hw,
2616                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2617                         if (status_mbx == FM10K_SUCCESS)
2618                                 PMD_INIT_LOG(INFO,
2619                                         "INT: Recreated Logical port");
2620                         else
2621                                 PMD_INIT_LOG(INFO,
2622                                         "INT: Logical ports weren't recreated");
2623
2624                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2625                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2626                         if (status_mbx != FM10K_SUCCESS)
2627                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2628
2629                         fm10k_mbx_unlock(hw);
2630
2631                         /* first clear the internal SW recording structure */
2632                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2633                                                 ETH_MQ_RX_VMDQ_FLAG))
2634                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2635                                         false);
2636
2637                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2638                                         MAIN_VSI_POOL_NUMBER);
2639
2640                         /*
2641                          * Add default mac address and vlan for the logical
2642                          * ports that have been created, leave to the
2643                          * application to fully recover Rx filtering.
2644                          */
2645                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2646                                         MAIN_VSI_POOL_NUMBER);
2647
2648                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2649                                                 ETH_MQ_RX_VMDQ_FLAG))
2650                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2651                                         true);
2652
2653                         dev_info->sm_down = 0;
2654                         _rte_eth_dev_callback_process(dev,
2655                                         RTE_ETH_EVENT_INTR_LSC,
2656                                         NULL);
2657                 }
2658         }
2659
2660         /* Handle mailbox message */
2661         fm10k_mbx_lock(hw);
2662         err = hw->mbx.ops.process(hw, &hw->mbx);
2663         fm10k_mbx_unlock(hw);
2664
2665         if (err == FM10K_ERR_RESET_REQUESTED) {
2666                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2667                 dev_info->sm_down = 1;
2668                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2669                                 NULL);
2670         }
2671
2672         /* Handle SRAM error */
2673         if (cause & FM10K_EICR_SRAMERROR) {
2674                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2675
2676                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2677                 /* Write to clear pending bits */
2678                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2679
2680                 /* Todo: print out error message after shared code  updates */
2681         }
2682
2683         /* Clear these 3 events if having any */
2684         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2685                  FM10K_EICR_SWITCHREADY;
2686         if (cause)
2687                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2688
2689         /* Re-enable interrupt from device side */
2690         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2691                                         FM10K_ITR_MASK_CLEAR);
2692         /* Re-enable interrupt from host side */
2693         rte_intr_ack(dev->intr_handle);
2694 }
2695
2696 /**
2697  * VF interrupt handler triggered by NIC for handling specific interrupt.
2698  *
2699  * @param handle
2700  *  Pointer to interrupt handle.
2701  * @param param
2702  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2703  *
2704  * @return
2705  *  void
2706  */
2707 static void
2708 fm10k_dev_interrupt_handler_vf(void *param)
2709 {
2710         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2711         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2712         struct fm10k_mbx_info *mbx = &hw->mbx;
2713         struct fm10k_dev_info *dev_info =
2714                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2715         const enum fm10k_mbx_state state = mbx->state;
2716         int status_mbx;
2717
2718         if (hw->mac.type != fm10k_mac_vf)
2719                 return;
2720
2721         /* Handle mailbox message if lock is acquired */
2722         fm10k_mbx_lock(hw);
2723         hw->mbx.ops.process(hw, &hw->mbx);
2724         fm10k_mbx_unlock(hw);
2725
2726         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2727                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2728
2729                 fm10k_mbx_lock(hw);
2730                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2731                                 MAX_LPORT_NUM, 1);
2732                 fm10k_mbx_unlock(hw);
2733
2734                 /* Setting reset flag */
2735                 dev_info->sm_down = 1;
2736                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2737                                 NULL);
2738         }
2739
2740         if (dev_info->sm_down == 1 &&
2741                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2742                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2743                 fm10k_mbx_lock(hw);
2744                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2745                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2746                 if (status_mbx != FM10K_SUCCESS)
2747                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2748                 fm10k_mbx_unlock(hw);
2749
2750                 /* first clear the internal SW recording structure */
2751                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2752                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2753                                 MAIN_VSI_POOL_NUMBER);
2754
2755                 /*
2756                  * Add default mac address and vlan for the logical ports that
2757                  * have been created, leave to the application to fully recover
2758                  * Rx filtering.
2759                  */
2760                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2761                                 MAIN_VSI_POOL_NUMBER);
2762                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2763
2764                 dev_info->sm_down = 0;
2765                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2766                                 NULL);
2767         }
2768
2769         /* Re-enable interrupt from device side */
2770         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2771                                         FM10K_ITR_MASK_CLEAR);
2772         /* Re-enable interrupt from host side */
2773         rte_intr_ack(dev->intr_handle);
2774 }
2775
2776 /* Mailbox message handler in VF */
2777 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2778         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2779         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2780         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2781         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2782 };
2783
2784 static int
2785 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2786 {
2787         int err = 0;
2788
2789         /* Initialize mailbox lock */
2790         fm10k_mbx_initlock(hw);
2791
2792         /* Replace default message handler with new ones */
2793         if (hw->mac.type == fm10k_mac_vf)
2794                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2795
2796         if (err) {
2797                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2798                                 err);
2799                 return err;
2800         }
2801         /* Connect to SM for PF device or PF for VF device */
2802         return hw->mbx.ops.connect(hw, &hw->mbx);
2803 }
2804
2805 static void
2806 fm10k_close_mbx_service(struct fm10k_hw *hw)
2807 {
2808         /* Disconnect from SM for PF device or PF for VF device */
2809         hw->mbx.ops.disconnect(hw, &hw->mbx);
2810 }
2811
2812 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2813         .dev_configure          = fm10k_dev_configure,
2814         .dev_start              = fm10k_dev_start,
2815         .dev_stop               = fm10k_dev_stop,
2816         .dev_close              = fm10k_dev_close,
2817         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2818         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2819         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2820         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2821         .stats_get              = fm10k_stats_get,
2822         .xstats_get             = fm10k_xstats_get,
2823         .xstats_get_names       = fm10k_xstats_get_names,
2824         .stats_reset            = fm10k_stats_reset,
2825         .xstats_reset           = fm10k_stats_reset,
2826         .link_update            = fm10k_link_update,
2827         .dev_infos_get          = fm10k_dev_infos_get,
2828         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2829         .vlan_filter_set        = fm10k_vlan_filter_set,
2830         .vlan_offload_set       = fm10k_vlan_offload_set,
2831         .mac_addr_add           = fm10k_macaddr_add,
2832         .mac_addr_remove        = fm10k_macaddr_remove,
2833         .rx_queue_start         = fm10k_dev_rx_queue_start,
2834         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2835         .tx_queue_start         = fm10k_dev_tx_queue_start,
2836         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2837         .rx_queue_setup         = fm10k_rx_queue_setup,
2838         .rx_queue_release       = fm10k_rx_queue_release,
2839         .tx_queue_setup         = fm10k_tx_queue_setup,
2840         .tx_queue_release       = fm10k_tx_queue_release,
2841         .rx_queue_count         = fm10k_dev_rx_queue_count,
2842         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2843         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2844         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2845         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2846         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2847         .reta_update            = fm10k_reta_update,
2848         .reta_query             = fm10k_reta_query,
2849         .rss_hash_update        = fm10k_rss_hash_update,
2850         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2851 };
2852
2853 static int ftag_check_handler(__rte_unused const char *key,
2854                 const char *value, __rte_unused void *opaque)
2855 {
2856         if (strcmp(value, "1"))
2857                 return -1;
2858
2859         return 0;
2860 }
2861
2862 static int
2863 fm10k_check_ftag(struct rte_devargs *devargs)
2864 {
2865         struct rte_kvargs *kvlist;
2866         const char *ftag_key = "enable_ftag";
2867
2868         if (devargs == NULL)
2869                 return 0;
2870
2871         kvlist = rte_kvargs_parse(devargs->args, NULL);
2872         if (kvlist == NULL)
2873                 return 0;
2874
2875         if (!rte_kvargs_count(kvlist, ftag_key)) {
2876                 rte_kvargs_free(kvlist);
2877                 return 0;
2878         }
2879         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2880         if (rte_kvargs_process(kvlist, ftag_key,
2881                                 ftag_check_handler, NULL) < 0) {
2882                 rte_kvargs_free(kvlist);
2883                 return 0;
2884         }
2885         rte_kvargs_free(kvlist);
2886
2887         return 1;
2888 }
2889
2890 static uint16_t
2891 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2892                     uint16_t nb_pkts)
2893 {
2894         uint16_t nb_tx = 0;
2895         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2896
2897         while (nb_pkts) {
2898                 uint16_t ret, num;
2899
2900                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2901                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2902                                                  num);
2903                 nb_tx += ret;
2904                 nb_pkts -= ret;
2905                 if (ret < num)
2906                         break;
2907         }
2908
2909         return nb_tx;
2910 }
2911
2912 static void __attribute__((cold))
2913 fm10k_set_tx_function(struct rte_eth_dev *dev)
2914 {
2915         struct fm10k_tx_queue *txq;
2916         int i;
2917         int use_sse = 1;
2918         uint16_t tx_ftag_en = 0;
2919
2920         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2921                 /* primary process has set the ftag flag and offloads */
2922                 txq = dev->data->tx_queues[0];
2923                 if (fm10k_tx_vec_condition_check(txq)) {
2924                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2925                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2926                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2927                 } else {
2928                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2929                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2930                         dev->tx_pkt_prepare = NULL;
2931                 }
2932                 return;
2933         }
2934
2935         if (fm10k_check_ftag(dev->device->devargs))
2936                 tx_ftag_en = 1;
2937
2938         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2939                 txq = dev->data->tx_queues[i];
2940                 txq->tx_ftag_en = tx_ftag_en;
2941                 /* Check if Vector Tx is satisfied */
2942                 if (fm10k_tx_vec_condition_check(txq))
2943                         use_sse = 0;
2944         }
2945
2946         if (use_sse) {
2947                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2948                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2949                         txq = dev->data->tx_queues[i];
2950                         fm10k_txq_vec_setup(txq);
2951                 }
2952                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2953                 dev->tx_pkt_prepare = NULL;
2954         } else {
2955                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2956                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2957                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2958         }
2959 }
2960
2961 static void __attribute__((cold))
2962 fm10k_set_rx_function(struct rte_eth_dev *dev)
2963 {
2964         struct fm10k_dev_info *dev_info =
2965                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2966         uint16_t i, rx_using_sse;
2967         uint16_t rx_ftag_en = 0;
2968
2969         if (fm10k_check_ftag(dev->device->devargs))
2970                 rx_ftag_en = 1;
2971
2972         /* In order to allow Vector Rx there are a few configuration
2973          * conditions to be met.
2974          */
2975         if (!fm10k_rx_vec_condition_check(dev) &&
2976                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2977                 if (dev->data->scattered_rx)
2978                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2979                 else
2980                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2981         } else if (dev->data->scattered_rx)
2982                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2983         else
2984                 dev->rx_pkt_burst = fm10k_recv_pkts;
2985
2986         rx_using_sse =
2987                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2988                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2989
2990         if (rx_using_sse)
2991                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2992         else
2993                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2994
2995         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2996                 return;
2997
2998         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2999                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3000
3001                 rxq->rx_using_sse = rx_using_sse;
3002                 rxq->rx_ftag_en = rx_ftag_en;
3003         }
3004 }
3005
3006 static void
3007 fm10k_params_init(struct rte_eth_dev *dev)
3008 {
3009         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3010         struct fm10k_dev_info *info =
3011                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3012
3013         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3014          * there is no way to get link status without reading BAR4.  Until this
3015          * works, assume we have maximum bandwidth.
3016          * @todo - fix bus info
3017          */
3018         hw->bus_caps.speed = fm10k_bus_speed_8000;
3019         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3020         hw->bus_caps.payload = fm10k_bus_payload_512;
3021         hw->bus.speed = fm10k_bus_speed_8000;
3022         hw->bus.width = fm10k_bus_width_pcie_x8;
3023         hw->bus.payload = fm10k_bus_payload_256;
3024
3025         info->rx_vec_allowed = true;
3026         info->sm_down = false;
3027 }
3028
3029 static int
3030 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3031 {
3032         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3033         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3034         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3035         int diag, i;
3036         struct fm10k_macvlan_filter_info *macvlan;
3037
3038         PMD_INIT_FUNC_TRACE();
3039
3040         dev->dev_ops = &fm10k_eth_dev_ops;
3041         dev->rx_pkt_burst = &fm10k_recv_pkts;
3042         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3043         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3044
3045         /*
3046          * Primary process does the whole initialization, for secondary
3047          * processes, we just select the same Rx and Tx function as primary.
3048          */
3049         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3050                 fm10k_set_rx_function(dev);
3051                 fm10k_set_tx_function(dev);
3052                 return 0;
3053         }
3054
3055         rte_eth_copy_pci_info(dev, pdev);
3056
3057         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3058         memset(macvlan, 0, sizeof(*macvlan));
3059         /* Vendor and Device ID need to be set before init of shared code */
3060         memset(hw, 0, sizeof(*hw));
3061         hw->device_id = pdev->id.device_id;
3062         hw->vendor_id = pdev->id.vendor_id;
3063         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3064         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3065         hw->revision_id = 0;
3066         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3067         if (hw->hw_addr == NULL) {
3068                 PMD_INIT_LOG(ERR, "Bad mem resource."
3069                         " Try to blacklist unused devices.");
3070                 return -EIO;
3071         }
3072
3073         /* Store fm10k_adapter pointer */
3074         hw->back = dev->data->dev_private;
3075
3076         /* Initialize the shared code */
3077         diag = fm10k_init_shared_code(hw);
3078         if (diag != FM10K_SUCCESS) {
3079                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3080                 return -EIO;
3081         }
3082
3083         /* Initialize parameters */
3084         fm10k_params_init(dev);
3085
3086         /* Initialize the hw */
3087         diag = fm10k_init_hw(hw);
3088         if (diag != FM10K_SUCCESS) {
3089                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3090                 return -EIO;
3091         }
3092
3093         /* Initialize MAC address(es) */
3094         dev->data->mac_addrs = rte_zmalloc("fm10k",
3095                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3096         if (dev->data->mac_addrs == NULL) {
3097                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3098                 return -ENOMEM;
3099         }
3100
3101         diag = fm10k_read_mac_addr(hw);
3102
3103         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3104                         &dev->data->mac_addrs[0]);
3105
3106         if (diag != FM10K_SUCCESS ||
3107                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3108
3109                 /* Generate a random addr */
3110                 rte_eth_random_addr(hw->mac.addr);
3111                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3112                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3113                 &dev->data->mac_addrs[0]);
3114         }
3115
3116         /* Reset the hw statistics */
3117         fm10k_stats_reset(dev);
3118
3119         /* Reset the hw */
3120         diag = fm10k_reset_hw(hw);
3121         if (diag != FM10K_SUCCESS) {
3122                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3123                 return -EIO;
3124         }
3125
3126         /* Setup mailbox service */
3127         diag = fm10k_setup_mbx_service(hw);
3128         if (diag != FM10K_SUCCESS) {
3129                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3130                 return -EIO;
3131         }
3132
3133         /*PF/VF has different interrupt handling mechanism */
3134         if (hw->mac.type == fm10k_mac_pf) {
3135                 /* register callback func to eal lib */
3136                 rte_intr_callback_register(intr_handle,
3137                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3138
3139                 /* enable MISC interrupt */
3140                 fm10k_dev_enable_intr_pf(dev);
3141         } else { /* VF */
3142                 rte_intr_callback_register(intr_handle,
3143                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3144
3145                 fm10k_dev_enable_intr_vf(dev);
3146         }
3147
3148         /* Enable intr after callback registered */
3149         rte_intr_enable(intr_handle);
3150
3151         hw->mac.ops.update_int_moderator(hw);
3152
3153         /* Make sure Switch Manager is ready before going forward. */
3154         if (hw->mac.type == fm10k_mac_pf) {
3155                 int switch_ready = 0;
3156
3157                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3158                         fm10k_mbx_lock(hw);
3159                         hw->mac.ops.get_host_state(hw, &switch_ready);
3160                         fm10k_mbx_unlock(hw);
3161                         if (switch_ready)
3162                                 break;
3163                         /* Delay some time to acquire async LPORT_MAP info. */
3164                         rte_delay_us(WAIT_SWITCH_MSG_US);
3165                 }
3166
3167                 if (switch_ready == 0) {
3168                         PMD_INIT_LOG(ERR, "switch is not ready");
3169                         return -1;
3170                 }
3171         }
3172
3173         /*
3174          * Below function will trigger operations on mailbox, acquire lock to
3175          * avoid race condition from interrupt handler. Operations on mailbox
3176          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3177          * will handle and generate an interrupt to our side. Then,  FIFO in
3178          * mailbox will be touched.
3179          */
3180         fm10k_mbx_lock(hw);
3181         /* Enable port first */
3182         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3183                                         MAX_LPORT_NUM, 1);
3184
3185         /* Set unicast mode by default. App can change to other mode in other
3186          * API func.
3187          */
3188         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3189                                         FM10K_XCAST_MODE_NONE);
3190
3191         fm10k_mbx_unlock(hw);
3192
3193         /* Make sure default VID is ready before going forward. */
3194         if (hw->mac.type == fm10k_mac_pf) {
3195                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3196                         if (hw->mac.default_vid)
3197                                 break;
3198                         /* Delay some time to acquire async port VLAN info. */
3199                         rte_delay_us(WAIT_SWITCH_MSG_US);
3200                 }
3201
3202                 if (!hw->mac.default_vid) {
3203                         PMD_INIT_LOG(ERR, "default VID is not ready");
3204                         return -1;
3205                 }
3206         }
3207
3208         /* Add default mac address */
3209         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3210                 MAIN_VSI_POOL_NUMBER);
3211
3212         return 0;
3213 }
3214
3215 static int
3216 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3217 {
3218         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3219         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3220         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3221         PMD_INIT_FUNC_TRACE();
3222
3223         /* only uninitialize in the primary process */
3224         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3225                 return 0;
3226
3227         /* safe to close dev here */
3228         fm10k_dev_close(dev);
3229
3230         dev->dev_ops = NULL;
3231         dev->rx_pkt_burst = NULL;
3232         dev->tx_pkt_burst = NULL;
3233
3234         /* disable uio/vfio intr */
3235         rte_intr_disable(intr_handle);
3236
3237         /*PF/VF has different interrupt handling mechanism */
3238         if (hw->mac.type == fm10k_mac_pf) {
3239                 /* disable interrupt */
3240                 fm10k_dev_disable_intr_pf(dev);
3241
3242                 /* unregister callback func to eal lib */
3243                 rte_intr_callback_unregister(intr_handle,
3244                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3245         } else {
3246                 /* disable interrupt */
3247                 fm10k_dev_disable_intr_vf(dev);
3248
3249                 rte_intr_callback_unregister(intr_handle,
3250                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3251         }
3252
3253         return 0;
3254 }
3255
3256 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3257         struct rte_pci_device *pci_dev)
3258 {
3259         return rte_eth_dev_pci_generic_probe(pci_dev,
3260                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3261 }
3262
3263 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3264 {
3265         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3266 }
3267
3268 /*
3269  * The set of PCI devices this driver supports. This driver will enable both PF
3270  * and SRIOV-VF devices.
3271  */
3272 static const struct rte_pci_id pci_id_fm10k_map[] = {
3273         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3274         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3275         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3276         { .vendor_id = 0, /* sentinel */ },
3277 };
3278
3279 static struct rte_pci_driver rte_pmd_fm10k = {
3280         .id_table = pci_id_fm10k_map,
3281         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3282         .probe = eth_fm10k_pci_probe,
3283         .remove = eth_fm10k_pci_remove,
3284 };
3285
3286 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3287 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3288 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3289
3290 RTE_INIT(fm10k_init_log)
3291 {
3292         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3293         if (fm10k_logtype_init >= 0)
3294                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3295         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3296         if (fm10k_logtype_driver >= 0)
3297                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3298 }