drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static int fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 __rte_weak int
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 __rte_weak uint16_t
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 __rte_weak uint16_t
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 __rte_weak int
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 __rte_weak void
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 __rte_weak void
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 __rte_weak int
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 __rte_weak uint16_t
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static int
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return 0;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS) {
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930                 return -EAGAIN;
 931         }
 932
 933         return 0;
 934 }
 935
 936 static int
 937 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 938 {
 939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 940         uint8_t mode;
 941         int status;
 942
 943         PMD_INIT_FUNC_TRACE();
 944
 945         /* Return if it didn't acquire valid glort range */
 946         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 947                 return 0;
 948
 949         if (dev->data->all_multicast == 1)
 950                 mode = FM10K_XCAST_MODE_ALLMULTI;
 951         else
 952                 mode = FM10K_XCAST_MODE_NONE;
 953
 954         fm10k_mbx_lock(hw);
 955         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 956                                 mode);
 957         fm10k_mbx_unlock(hw);
 958
 959         if (status != FM10K_SUCCESS) {
 960                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 961                 return -EAGAIN;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static int
 968 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971         int status;
 972
 973         PMD_INIT_FUNC_TRACE();
 974
 975         /* Return if it didn't acquire valid glort range */
 976         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 977                 return 0;
 978
 979         /* If promiscuous mode is enabled, it doesn't make sense to enable
 980          * allmulticast and disable promiscuous since fm10k only can select
 981          * one of the modes.
 982          */
 983         if (dev->data->promiscuous) {
 984                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 985                         "needn't enable allmulticast");
 986                 return 0;
 987         }
 988
 989         fm10k_mbx_lock(hw);
 990         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 991                                 FM10K_XCAST_MODE_ALLMULTI);
 992         fm10k_mbx_unlock(hw);
 993
 994         if (status != FM10K_SUCCESS) {
 995                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 996                 return -EAGAIN;
 997         }
 998
 999         return 0;
1000 }
1001
1002 static int
1003 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1004 {
1005         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1006         int status;
1007
1008         PMD_INIT_FUNC_TRACE();
1009
1010         /* Return if it didn't acquire valid glort range */
1011         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1012                 return 0;
1013
1014         if (dev->data->promiscuous) {
1015                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1016                         "since promisc mode is enabled");
1017                 return -EINVAL;
1018         }
1019
1020         fm10k_mbx_lock(hw);
1021         /* Change mode to unicast mode */
1022         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1023                                 FM10K_XCAST_MODE_NONE);
1024         fm10k_mbx_unlock(hw);
1025
1026         if (status != FM10K_SUCCESS) {
1027                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1028                 return -EAGAIN;
1029         }
1030
1031         return 0;
1032 }
1033
1034 static void
1035 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1036 {
1037         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1038         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1039         uint16_t nb_queue_pools;
1040         struct fm10k_macvlan_filter_info *macvlan;
1041
1042         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1043         nb_queue_pools = macvlan->nb_queue_pools;
1044         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1045         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1046
1047         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1048         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1049         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1050                         hw->mac.dglort_map;
1051         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1052         /* Configure VMDQ/RSS DGlort Decoder */
1053         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1054
1055         /* Flow Director configurations, only queue number is valid. */
1056         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1057         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1058                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1059         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1060         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1061
1062         /* Invalidate all other GLORT entries */
1063         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1064                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1065                                 FM10K_DGLORTMAP_NONE);
1066 }
1067
1068 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1069 static int
1070 fm10k_dev_start(struct rte_eth_dev *dev)
1071 {
1072         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1073         int i, diag;
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         /* stop, init, then start the hw */
1078         diag = fm10k_stop_hw(hw);
1079         if (diag != FM10K_SUCCESS) {
1080                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1081                 return -EIO;
1082         }
1083
1084         diag = fm10k_init_hw(hw);
1085         if (diag != FM10K_SUCCESS) {
1086                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1087                 return -EIO;
1088         }
1089
1090         diag = fm10k_start_hw(hw);
1091         if (diag != FM10K_SUCCESS) {
1092                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1093                 return -EIO;
1094         }
1095
1096         diag = fm10k_dev_tx_init(dev);
1097         if (diag) {
1098                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1099                 return diag;
1100         }
1101
1102         if (fm10k_dev_rxq_interrupt_setup(dev))
1103                 return -EIO;
1104
1105         diag = fm10k_dev_rx_init(dev);
1106         if (diag) {
1107                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1108                 return diag;
1109         }
1110
1111         if (hw->mac.type == fm10k_mac_pf)
1112                 fm10k_dev_dglort_map_configure(dev);
1113
1114         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1115                 struct fm10k_rx_queue *rxq;
1116                 rxq = dev->data->rx_queues[i];
1117
1118                 if (rxq->rx_deferred_start)
1119                         continue;
1120                 diag = fm10k_dev_rx_queue_start(dev, i);
1121                 if (diag != 0) {
1122                         int j;
1123                         for (j = 0; j < i; ++j)
1124                                 rx_queue_clean(dev->data->rx_queues[j]);
1125                         return diag;
1126                 }
1127         }
1128
1129         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1130                 struct fm10k_tx_queue *txq;
1131                 txq = dev->data->tx_queues[i];
1132
1133                 if (txq->tx_deferred_start)
1134                         continue;
1135                 diag = fm10k_dev_tx_queue_start(dev, i);
1136                 if (diag != 0) {
1137                         int j;
1138                         for (j = 0; j < i; ++j)
1139                                 tx_queue_clean(dev->data->tx_queues[j]);
1140                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1141                                 rx_queue_clean(dev->data->rx_queues[j]);
1142                         return diag;
1143                 }
1144         }
1145
1146         /* Update default vlan when not in VMDQ mode */
1147         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1148                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1149
1150         fm10k_link_update(dev, 0);
1151
1152         return 0;
1153 }
1154
1155 static void
1156 fm10k_dev_stop(struct rte_eth_dev *dev)
1157 {
1158         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1159         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1160         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1161         int i;
1162
1163         PMD_INIT_FUNC_TRACE();
1164
1165         if (dev->data->tx_queues)
1166                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1167                         fm10k_dev_tx_queue_stop(dev, i);
1168
1169         if (dev->data->rx_queues)
1170                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1171                         fm10k_dev_rx_queue_stop(dev, i);
1172
1173         /* Disable datapath event */
1174         if (rte_intr_dp_is_en(intr_handle)) {
1175                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1176                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1177                                 3 << FM10K_RXINT_TIMER_SHIFT);
1178                         if (hw->mac.type == fm10k_mac_pf)
1179                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1180                                         FM10K_ITR_MASK_SET);
1181                         else
1182                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1183                                         FM10K_ITR_MASK_SET);
1184                 }
1185         }
1186         /* Clean datapath event and queue/vec mapping */
1187         rte_intr_efd_disable(intr_handle);
1188         rte_free(intr_handle->intr_vec);
1189         intr_handle->intr_vec = NULL;
1190 }
1191
1192 static void
1193 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1194 {
1195         int i;
1196
1197         PMD_INIT_FUNC_TRACE();
1198
1199         if (dev->data->tx_queues) {
1200                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1201                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1202
1203                         tx_queue_free(txq);
1204                 }
1205         }
1206
1207         if (dev->data->rx_queues) {
1208                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1209                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1210         }
1211 }
1212
1213 static int
1214 fm10k_link_update(struct rte_eth_dev *dev,
1215         __rte_unused int wait_to_complete)
1216 {
1217         struct fm10k_dev_info *dev_info =
1218                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1219         PMD_INIT_FUNC_TRACE();
1220
1221         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1222         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1223         dev->data->dev_link.link_status =
1224                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1225         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1226
1227         return 0;
1228 }
1229
1230 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1231         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1232 {
1233         unsigned i, q;
1234         unsigned count = 0;
1235
1236         if (xstats_names != NULL) {
1237                 /* Note: limit checked in rte_eth_xstats_names() */
1238
1239                 /* Global stats */
1240                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1241                         snprintf(xstats_names[count].name,
1242                                 sizeof(xstats_names[count].name),
1243                                 "%s", fm10k_hw_stats_strings[count].name);
1244                         count++;
1245                 }
1246
1247                 /* PF queue stats */
1248                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1249                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1250                                 snprintf(xstats_names[count].name,
1251                                         sizeof(xstats_names[count].name),
1252                                         "rx_q%u_%s", q,
1253                                         fm10k_hw_stats_rx_q_strings[i].name);
1254                                 count++;
1255                         }
1256                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1257                                 snprintf(xstats_names[count].name,
1258                                         sizeof(xstats_names[count].name),
1259                                         "tx_q%u_%s", q,
1260                                         fm10k_hw_stats_tx_q_strings[i].name);
1261                                 count++;
1262                         }
1263                 }
1264         }
1265         return FM10K_NB_XSTATS;
1266 }
1267
1268 static int
1269 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1270                  unsigned n)
1271 {
1272         struct fm10k_hw_stats *hw_stats =
1273                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1274         unsigned i, q, count = 0;
1275
1276         if (n < FM10K_NB_XSTATS)
1277                 return FM10K_NB_XSTATS;
1278
1279         /* Global stats */
1280         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1281                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1282                         fm10k_hw_stats_strings[count].offset);
1283                 xstats[count].id = count;
1284                 count++;
1285         }
1286
1287         /* PF queue stats */
1288         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1289                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1290                         xstats[count].value =
1291                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1292                                 fm10k_hw_stats_rx_q_strings[i].offset);
1293                         xstats[count].id = count;
1294                         count++;
1295                 }
1296                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1297                         xstats[count].value =
1298                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1299                                 fm10k_hw_stats_tx_q_strings[i].offset);
1300                         xstats[count].id = count;
1301                         count++;
1302                 }
1303         }
1304
1305         return FM10K_NB_XSTATS;
1306 }
1307
1308 static int
1309 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1310 {
1311         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1312         struct fm10k_hw *hw =
1313                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1314         struct fm10k_hw_stats *hw_stats =
1315                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1316         int i;
1317
1318         PMD_INIT_FUNC_TRACE();
1319
1320         fm10k_update_hw_stats(hw, hw_stats);
1321
1322         ipackets = opackets = ibytes = obytes = imissed = 0;
1323         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1324                 (i < hw->mac.max_queues); ++i) {
1325                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1326                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1327                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1328                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1329                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1330                 ipackets += stats->q_ipackets[i];
1331                 opackets += stats->q_opackets[i];
1332                 ibytes   += stats->q_ibytes[i];
1333                 obytes   += stats->q_obytes[i];
1334                 imissed  += stats->q_errors[i];
1335         }
1336         stats->ipackets = ipackets;
1337         stats->opackets = opackets;
1338         stats->ibytes = ibytes;
1339         stats->obytes = obytes;
1340         stats->imissed = imissed;
1341         return 0;
1342 }
1343
1344 static int
1345 fm10k_stats_reset(struct rte_eth_dev *dev)
1346 {
1347         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1348         struct fm10k_hw_stats *hw_stats =
1349                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1350
1351         PMD_INIT_FUNC_TRACE();
1352
1353         memset(hw_stats, 0, sizeof(*hw_stats));
1354         fm10k_rebind_hw_stats(hw, hw_stats);
1355
1356         return 0;
1357 }
1358
1359 static int
1360 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1361         struct rte_eth_dev_info *dev_info)
1362 {
1363         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1364         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1365
1366         PMD_INIT_FUNC_TRACE();
1367
1368         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1369         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1370         dev_info->max_rx_queues      = hw->mac.max_queues;
1371         dev_info->max_tx_queues      = hw->mac.max_queues;
1372         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1373         dev_info->max_hash_mac_addrs = 0;
1374         dev_info->max_vfs            = pdev->max_vfs;
1375         dev_info->vmdq_pool_base     = 0;
1376         dev_info->vmdq_queue_base    = 0;
1377         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1378         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1379         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1380         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1381                                     dev_info->rx_queue_offload_capa;
1382         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1383         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1384                                     dev_info->tx_queue_offload_capa;
1385
1386         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1387         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1388         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1389                                         ETH_RSS_IPV6 |
1390                                         ETH_RSS_IPV6_EX |
1391                                         ETH_RSS_NONFRAG_IPV4_TCP |
1392                                         ETH_RSS_NONFRAG_IPV6_TCP |
1393                                         ETH_RSS_IPV6_TCP_EX |
1394                                         ETH_RSS_NONFRAG_IPV4_UDP |
1395                                         ETH_RSS_NONFRAG_IPV6_UDP |
1396                                         ETH_RSS_IPV6_UDP_EX;
1397
1398         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1399                 .rx_thresh = {
1400                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1401                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1402                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1403                 },
1404                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1405                 .rx_drop_en = 0,
1406                 .offloads = 0,
1407         };
1408
1409         dev_info->default_txconf = (struct rte_eth_txconf) {
1410                 .tx_thresh = {
1411                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1412                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1413                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1414                 },
1415                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1416                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1417                 .offloads = 0,
1418         };
1419
1420         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1421                 .nb_max = FM10K_MAX_RX_DESC,
1422                 .nb_min = FM10K_MIN_RX_DESC,
1423                 .nb_align = FM10K_MULT_RX_DESC,
1424         };
1425
1426         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1427                 .nb_max = FM10K_MAX_TX_DESC,
1428                 .nb_min = FM10K_MIN_TX_DESC,
1429                 .nb_align = FM10K_MULT_TX_DESC,
1430                 .nb_seg_max = FM10K_TX_MAX_SEG,
1431                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1432         };
1433
1434         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1435                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1436                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1437
1438         return 0;
1439 }
1440
1441 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1442 static const uint32_t *
1443 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1444 {
1445         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1446             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1447                 static uint32_t ptypes[] = {
1448                         /* refers to rx_desc_to_ol_flags() */
1449                         RTE_PTYPE_L2_ETHER,
1450                         RTE_PTYPE_L3_IPV4,
1451                         RTE_PTYPE_L3_IPV4_EXT,
1452                         RTE_PTYPE_L3_IPV6,
1453                         RTE_PTYPE_L3_IPV6_EXT,
1454                         RTE_PTYPE_L4_TCP,
1455                         RTE_PTYPE_L4_UDP,
1456                         RTE_PTYPE_UNKNOWN
1457                 };
1458
1459                 return ptypes;
1460         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1461                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1462                 static uint32_t ptypes_vec[] = {
1463                         /* refers to fm10k_desc_to_pktype_v() */
1464                         RTE_PTYPE_L3_IPV4,
1465                         RTE_PTYPE_L3_IPV4_EXT,
1466                         RTE_PTYPE_L3_IPV6,
1467                         RTE_PTYPE_L3_IPV6_EXT,
1468                         RTE_PTYPE_L4_TCP,
1469                         RTE_PTYPE_L4_UDP,
1470                         RTE_PTYPE_TUNNEL_GENEVE,
1471                         RTE_PTYPE_TUNNEL_NVGRE,
1472                         RTE_PTYPE_TUNNEL_VXLAN,
1473                         RTE_PTYPE_TUNNEL_GRE,
1474                         RTE_PTYPE_UNKNOWN
1475                 };
1476
1477                 return ptypes_vec;
1478         }
1479
1480         return NULL;
1481 }
1482 #else
1483 static const uint32_t *
1484 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1485 {
1486         return NULL;
1487 }
1488 #endif
1489
1490 static int
1491 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1492 {
1493         s32 result;
1494         uint16_t mac_num = 0;
1495         uint32_t vid_idx, vid_bit, mac_index;
1496         struct fm10k_hw *hw;
1497         struct fm10k_macvlan_filter_info *macvlan;
1498         struct rte_eth_dev_data *data = dev->data;
1499
1500         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1501         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1502
1503         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1504                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1505                 return -EINVAL;
1506         }
1507
1508         if (vlan_id > ETH_VLAN_ID_MAX) {
1509                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1510                 return -EINVAL;
1511         }
1512
1513         vid_idx = FM10K_VFTA_IDX(vlan_id);
1514         vid_bit = FM10K_VFTA_BIT(vlan_id);
1515         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1516         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1517                 return 0;
1518         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1519         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1520                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1521                         "in the VLAN filter table");
1522                 return -EINVAL;
1523         }
1524
1525         fm10k_mbx_lock(hw);
1526         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1527         fm10k_mbx_unlock(hw);
1528         if (result != FM10K_SUCCESS) {
1529                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1530                 return -EIO;
1531         }
1532
1533         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1534                         (result == FM10K_SUCCESS); mac_index++) {
1535                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1536                         continue;
1537                 if (mac_num > macvlan->mac_num - 1) {
1538                         PMD_INIT_LOG(ERR, "MAC address number "
1539                                         "not match");
1540                         break;
1541                 }
1542                 fm10k_mbx_lock(hw);
1543                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1544                         data->mac_addrs[mac_index].addr_bytes,
1545                         vlan_id, on, 0);
1546                 fm10k_mbx_unlock(hw);
1547                 mac_num++;
1548         }
1549         if (result != FM10K_SUCCESS) {
1550                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1551                 return -EIO;
1552         }
1553
1554         if (on) {
1555                 macvlan->vlan_num++;
1556                 macvlan->vfta[vid_idx] |= vid_bit;
1557         } else {
1558                 macvlan->vlan_num--;
1559                 macvlan->vfta[vid_idx] &= ~vid_bit;
1560         }
1561         return 0;
1562 }
1563
1564 static int
1565 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1566 {
1567         if (mask & ETH_VLAN_STRIP_MASK) {
1568                 if (!(dev->data->dev_conf.rxmode.offloads &
1569                         DEV_RX_OFFLOAD_VLAN_STRIP))
1570                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1571                                         "always on in fm10k");
1572         }
1573
1574         if (mask & ETH_VLAN_EXTEND_MASK) {
1575                 if (dev->data->dev_conf.rxmode.offloads &
1576                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1577                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1578                                         "supported in fm10k");
1579         }
1580
1581         if (mask & ETH_VLAN_FILTER_MASK) {
1582                 if (!(dev->data->dev_conf.rxmode.offloads &
1583                         DEV_RX_OFFLOAD_VLAN_FILTER))
1584                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1585         }
1586
1587         return 0;
1588 }
1589
1590 /* Add/Remove a MAC address, and update filters to main VSI */
1591 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1592                 const u8 *mac, bool add, uint32_t pool)
1593 {
1594         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1595         struct fm10k_macvlan_filter_info *macvlan;
1596         uint32_t i, j, k;
1597
1598         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1599
1600         if (pool != MAIN_VSI_POOL_NUMBER) {
1601                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1602                         "mac to pool %u", pool);
1603                 return;
1604         }
1605         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1606                 if (!macvlan->vfta[j])
1607                         continue;
1608                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1609                         if (!(macvlan->vfta[j] & (1 << k)))
1610                                 continue;
1611                         if (i + 1 > macvlan->vlan_num) {
1612                                 PMD_INIT_LOG(ERR, "vlan number not match");
1613                                 return;
1614                         }
1615                         fm10k_mbx_lock(hw);
1616                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1617                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1618                         fm10k_mbx_unlock(hw);
1619                         i++;
1620                 }
1621         }
1622 }
1623
1624 /* Add/Remove a MAC address, and update filters to VMDQ */
1625 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1626                 const u8 *mac, bool add, uint32_t pool)
1627 {
1628         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1629         struct fm10k_macvlan_filter_info *macvlan;
1630         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1631         uint32_t i;
1632
1633         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1634         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1635
1636         if (pool > macvlan->nb_queue_pools) {
1637                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1638                         " Max pool is %u",
1639                         pool, macvlan->nb_queue_pools);
1640                 return;
1641         }
1642         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1643                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1644                         continue;
1645                 fm10k_mbx_lock(hw);
1646                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1647                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1648                 fm10k_mbx_unlock(hw);
1649         }
1650 }
1651
1652 /* Add/Remove a MAC address, and update filters */
1653 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1654                 const u8 *mac, bool add, uint32_t pool)
1655 {
1656         struct fm10k_macvlan_filter_info *macvlan;
1657
1658         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1659
1660         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1661                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1662         else
1663                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1664
1665         if (add)
1666                 macvlan->mac_num++;
1667         else
1668                 macvlan->mac_num--;
1669 }
1670
1671 /* Add a MAC address, and update filters */
1672 static int
1673 fm10k_macaddr_add(struct rte_eth_dev *dev,
1674                 struct rte_ether_addr *mac_addr,
1675                 uint32_t index,
1676                 uint32_t pool)
1677 {
1678         struct fm10k_macvlan_filter_info *macvlan;
1679
1680         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1681         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1682         macvlan->mac_vmdq_id[index] = pool;
1683         return 0;
1684 }
1685
1686 /* Remove a MAC address, and update filters */
1687 static void
1688 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1689 {
1690         struct rte_eth_dev_data *data = dev->data;
1691         struct fm10k_macvlan_filter_info *macvlan;
1692
1693         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1694         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1695                         FALSE, macvlan->mac_vmdq_id[index]);
1696         macvlan->mac_vmdq_id[index] = 0;
1697 }
1698
1699 static inline int
1700 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1701 {
1702         if ((request < min) || (request > max) || ((request % mult) != 0))
1703                 return -1;
1704         else
1705                 return 0;
1706 }
1707
1708
1709 static inline int
1710 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1711 {
1712         if ((request < min) || (request > max) || ((div % request) != 0))
1713                 return -1;
1714         else
1715                 return 0;
1716 }
1717
1718 static inline int
1719 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1720 {
1721         uint16_t rx_free_thresh;
1722
1723         if (conf->rx_free_thresh == 0)
1724                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1725         else
1726                 rx_free_thresh = conf->rx_free_thresh;
1727
1728         /* make sure the requested threshold satisfies the constraints */
1729         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1730                         FM10K_RX_FREE_THRESH_MAX(q),
1731                         FM10K_RX_FREE_THRESH_DIV(q),
1732                         rx_free_thresh)) {
1733                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1734                         "less than or equal to %u, "
1735                         "greater than or equal to %u, "
1736                         "and a divisor of %u",
1737                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1738                         FM10K_RX_FREE_THRESH_MIN(q),
1739                         FM10K_RX_FREE_THRESH_DIV(q));
1740                 return -EINVAL;
1741         }
1742
1743         q->alloc_thresh = rx_free_thresh;
1744         q->drop_en = conf->rx_drop_en;
1745         q->rx_deferred_start = conf->rx_deferred_start;
1746
1747         return 0;
1748 }
1749
1750 /*
1751  * Hardware requires specific alignment for Rx packet buffers. At
1752  * least one of the following two conditions must be satisfied.
1753  *  1. Address is 512B aligned
1754  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1755  *
1756  * As such, the driver may need to adjust the DMA address within the
1757  * buffer by up to 512B.
1758  *
1759  * return 1 if the element size is valid, otherwise return 0.
1760  */
1761 static int
1762 mempool_element_size_valid(struct rte_mempool *mp)
1763 {
1764         uint32_t min_size;
1765
1766         /* elt_size includes mbuf header and headroom */
1767         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1768                         RTE_PKTMBUF_HEADROOM;
1769
1770         /* account for up to 512B of alignment */
1771         min_size -= FM10K_RX_DATABUF_ALIGN;
1772
1773         /* sanity check for overflow */
1774         if (min_size > mp->elt_size)
1775                 return 0;
1776
1777         /* size is valid */
1778         return 1;
1779 }
1780
1781 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1782 {
1783         RTE_SET_USED(dev);
1784
1785         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1786 }
1787
1788 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1789 {
1790         RTE_SET_USED(dev);
1791
1792         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1793                            DEV_RX_OFFLOAD_VLAN_FILTER |
1794                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1795                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1796                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1797                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1798                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1799 }
1800
1801 static int
1802 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1803         uint16_t nb_desc, unsigned int socket_id,
1804         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1805 {
1806         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1807         struct fm10k_dev_info *dev_info =
1808                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1809         struct fm10k_rx_queue *q;
1810         const struct rte_memzone *mz;
1811         uint64_t offloads;
1812
1813         PMD_INIT_FUNC_TRACE();
1814
1815         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1816
1817         /* make sure the mempool element size can account for alignment. */
1818         if (!mempool_element_size_valid(mp)) {
1819                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1820                 return -EINVAL;
1821         }
1822
1823         /* make sure a valid number of descriptors have been requested */
1824         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1825                                 FM10K_MULT_RX_DESC, nb_desc)) {
1826                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1827                         "less than or equal to %"PRIu32", "
1828                         "greater than or equal to %u, "
1829                         "and a multiple of %u",
1830                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1831                         FM10K_MULT_RX_DESC);
1832                 return -EINVAL;
1833         }
1834
1835         /*
1836          * if this queue existed already, free the associated memory. The
1837          * queue cannot be reused in case we need to allocate memory on
1838          * different socket than was previously used.
1839          */
1840         if (dev->data->rx_queues[queue_id] != NULL) {
1841                 rx_queue_free(dev->data->rx_queues[queue_id]);
1842                 dev->data->rx_queues[queue_id] = NULL;
1843         }
1844
1845         /* allocate memory for the queue structure */
1846         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1847                                 socket_id);
1848         if (q == NULL) {
1849                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1850                 return -ENOMEM;
1851         }
1852
1853         /* setup queue */
1854         q->mp = mp;
1855         q->nb_desc = nb_desc;
1856         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1857         q->port_id = dev->data->port_id;
1858         q->queue_id = queue_id;
1859         q->tail_ptr = (volatile uint32_t *)
1860                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1861         q->offloads = offloads;
1862         if (handle_rxconf(q, conf))
1863                 return -EINVAL;
1864
1865         /* allocate memory for the software ring */
1866         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1867                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1868                         RTE_CACHE_LINE_SIZE, socket_id);
1869         if (q->sw_ring == NULL) {
1870                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1871                 rte_free(q);
1872                 return -ENOMEM;
1873         }
1874
1875         /*
1876          * allocate memory for the hardware descriptor ring. A memzone large
1877          * enough to hold the maximum ring size is requested to allow for
1878          * resizing in later calls to the queue setup function.
1879          */
1880         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1881                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1882                                       socket_id);
1883         if (mz == NULL) {
1884                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1885                 rte_free(q->sw_ring);
1886                 rte_free(q);
1887                 return -ENOMEM;
1888         }
1889         q->hw_ring = mz->addr;
1890         q->hw_ring_phys_addr = mz->iova;
1891
1892         /* Check if number of descs satisfied Vector requirement */
1893         if (!rte_is_power_of_2(nb_desc)) {
1894                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1895                                     "preconditions - canceling the feature for "
1896                                     "the whole port[%d]",
1897                              q->queue_id, q->port_id);
1898                 dev_info->rx_vec_allowed = false;
1899         } else
1900                 fm10k_rxq_vec_setup(q);
1901
1902         dev->data->rx_queues[queue_id] = q;
1903         return 0;
1904 }
1905
1906 static void
1907 fm10k_rx_queue_release(void *queue)
1908 {
1909         PMD_INIT_FUNC_TRACE();
1910
1911         rx_queue_free(queue);
1912 }
1913
1914 static inline int
1915 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1916 {
1917         uint16_t tx_free_thresh;
1918         uint16_t tx_rs_thresh;
1919
1920         /* constraint MACROs require that tx_free_thresh is configured
1921          * before tx_rs_thresh */
1922         if (conf->tx_free_thresh == 0)
1923                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1924         else
1925                 tx_free_thresh = conf->tx_free_thresh;
1926
1927         /* make sure the requested threshold satisfies the constraints */
1928         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1929                         FM10K_TX_FREE_THRESH_MAX(q),
1930                         FM10K_TX_FREE_THRESH_DIV(q),
1931                         tx_free_thresh)) {
1932                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1933                         "less than or equal to %u, "
1934                         "greater than or equal to %u, "
1935                         "and a divisor of %u",
1936                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1937                         FM10K_TX_FREE_THRESH_MIN(q),
1938                         FM10K_TX_FREE_THRESH_DIV(q));
1939                 return -EINVAL;
1940         }
1941
1942         q->free_thresh = tx_free_thresh;
1943
1944         if (conf->tx_rs_thresh == 0)
1945                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1946         else
1947                 tx_rs_thresh = conf->tx_rs_thresh;
1948
1949         q->tx_deferred_start = conf->tx_deferred_start;
1950
1951         /* make sure the requested threshold satisfies the constraints */
1952         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1953                         FM10K_TX_RS_THRESH_MAX(q),
1954                         FM10K_TX_RS_THRESH_DIV(q),
1955                         tx_rs_thresh)) {
1956                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1957                         "less than or equal to %u, "
1958                         "greater than or equal to %u, "
1959                         "and a divisor of %u",
1960                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1961                         FM10K_TX_RS_THRESH_MIN(q),
1962                         FM10K_TX_RS_THRESH_DIV(q));
1963                 return -EINVAL;
1964         }
1965
1966         q->rs_thresh = tx_rs_thresh;
1967
1968         return 0;
1969 }
1970
1971 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1972 {
1973         RTE_SET_USED(dev);
1974
1975         return 0;
1976 }
1977
1978 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1979 {
1980         RTE_SET_USED(dev);
1981
1982         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1983                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1984                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1985                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1986                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1987                           DEV_TX_OFFLOAD_TCP_TSO);
1988 }
1989
1990 static int
1991 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1992         uint16_t nb_desc, unsigned int socket_id,
1993         const struct rte_eth_txconf *conf)
1994 {
1995         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1996         struct fm10k_tx_queue *q;
1997         const struct rte_memzone *mz;
1998         uint64_t offloads;
1999
2000         PMD_INIT_FUNC_TRACE();
2001
2002         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2003
2004         /* make sure a valid number of descriptors have been requested */
2005         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2006                                 FM10K_MULT_TX_DESC, nb_desc)) {
2007                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2008                         "less than or equal to %"PRIu32", "
2009                         "greater than or equal to %u, "
2010                         "and a multiple of %u",
2011                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2012                         FM10K_MULT_TX_DESC);
2013                 return -EINVAL;
2014         }
2015
2016         /*
2017          * if this queue existed already, free the associated memory. The
2018          * queue cannot be reused in case we need to allocate memory on
2019          * different socket than was previously used.
2020          */
2021         if (dev->data->tx_queues[queue_id] != NULL) {
2022                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2023
2024                 tx_queue_free(txq);
2025                 dev->data->tx_queues[queue_id] = NULL;
2026         }
2027
2028         /* allocate memory for the queue structure */
2029         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2030                                 socket_id);
2031         if (q == NULL) {
2032                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2033                 return -ENOMEM;
2034         }
2035
2036         /* setup queue */
2037         q->nb_desc = nb_desc;
2038         q->port_id = dev->data->port_id;
2039         q->queue_id = queue_id;
2040         q->offloads = offloads;
2041         q->ops = &def_txq_ops;
2042         q->tail_ptr = (volatile uint32_t *)
2043                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2044         if (handle_txconf(q, conf))
2045                 return -EINVAL;
2046
2047         /* allocate memory for the software ring */
2048         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2049                                         nb_desc * sizeof(struct rte_mbuf *),
2050                                         RTE_CACHE_LINE_SIZE, socket_id);
2051         if (q->sw_ring == NULL) {
2052                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2053                 rte_free(q);
2054                 return -ENOMEM;
2055         }
2056
2057         /*
2058          * allocate memory for the hardware descriptor ring. A memzone large
2059          * enough to hold the maximum ring size is requested to allow for
2060          * resizing in later calls to the queue setup function.
2061          */
2062         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2063                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2064                                       socket_id);
2065         if (mz == NULL) {
2066                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2067                 rte_free(q->sw_ring);
2068                 rte_free(q);
2069                 return -ENOMEM;
2070         }
2071         q->hw_ring = mz->addr;
2072         q->hw_ring_phys_addr = mz->iova;
2073
2074         /*
2075          * allocate memory for the RS bit tracker. Enough slots to hold the
2076          * descriptor index for each RS bit needing to be set are required.
2077          */
2078         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2079                                 ((nb_desc + 1) / q->rs_thresh) *
2080                                 sizeof(uint16_t),
2081                                 RTE_CACHE_LINE_SIZE, socket_id);
2082         if (q->rs_tracker.list == NULL) {
2083                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2084                 rte_free(q->sw_ring);
2085                 rte_free(q);
2086                 return -ENOMEM;
2087         }
2088
2089         dev->data->tx_queues[queue_id] = q;
2090         return 0;
2091 }
2092
2093 static void
2094 fm10k_tx_queue_release(void *queue)
2095 {
2096         struct fm10k_tx_queue *q = queue;
2097         PMD_INIT_FUNC_TRACE();
2098
2099         tx_queue_free(q);
2100 }
2101
2102 static int
2103 fm10k_reta_update(struct rte_eth_dev *dev,
2104                         struct rte_eth_rss_reta_entry64 *reta_conf,
2105                         uint16_t reta_size)
2106 {
2107         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2108         uint16_t i, j, idx, shift;
2109         uint8_t mask;
2110         uint32_t reta;
2111
2112         PMD_INIT_FUNC_TRACE();
2113
2114         if (reta_size > FM10K_MAX_RSS_INDICES) {
2115                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2116                         "(%d) doesn't match the number hardware can supported "
2117                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2118                 return -EINVAL;
2119         }
2120
2121         /*
2122          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2123          * 128-entries in 32 registers
2124          */
2125         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2126                 idx = i / RTE_RETA_GROUP_SIZE;
2127                 shift = i % RTE_RETA_GROUP_SIZE;
2128                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2129                                 BIT_MASK_PER_UINT32);
2130                 if (mask == 0)
2131                         continue;
2132
2133                 reta = 0;
2134                 if (mask != BIT_MASK_PER_UINT32)
2135                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2136
2137                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2138                         if (mask & (0x1 << j)) {
2139                                 if (mask != 0xF)
2140                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2141                                 reta |= reta_conf[idx].reta[shift + j] <<
2142                                                 (CHAR_BIT * j);
2143                         }
2144                 }
2145                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2146         }
2147
2148         return 0;
2149 }
2150
2151 static int
2152 fm10k_reta_query(struct rte_eth_dev *dev,
2153                         struct rte_eth_rss_reta_entry64 *reta_conf,
2154                         uint16_t reta_size)
2155 {
2156         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2157         uint16_t i, j, idx, shift;
2158         uint8_t mask;
2159         uint32_t reta;
2160
2161         PMD_INIT_FUNC_TRACE();
2162
2163         if (reta_size < FM10K_MAX_RSS_INDICES) {
2164                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2165                         "(%d) doesn't match the number hardware can supported "
2166                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2167                 return -EINVAL;
2168         }
2169
2170         /*
2171          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2172          * 128-entries in 32 registers
2173          */
2174         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2175                 idx = i / RTE_RETA_GROUP_SIZE;
2176                 shift = i % RTE_RETA_GROUP_SIZE;
2177                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2178                                 BIT_MASK_PER_UINT32);
2179                 if (mask == 0)
2180                         continue;
2181
2182                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2183                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2184                         if (mask & (0x1 << j))
2185                                 reta_conf[idx].reta[shift + j] = ((reta >>
2186                                         CHAR_BIT * j) & UINT8_MAX);
2187                 }
2188         }
2189
2190         return 0;
2191 }
2192
2193 static int
2194 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2195         struct rte_eth_rss_conf *rss_conf)
2196 {
2197         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2198         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2199         uint32_t mrqc;
2200         uint64_t hf = rss_conf->rss_hf;
2201         int i;
2202
2203         PMD_INIT_FUNC_TRACE();
2204
2205         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2206                                 FM10K_RSSRK_ENTRIES_PER_REG))
2207                 return -EINVAL;
2208
2209         if (hf == 0)
2210                 return -EINVAL;
2211
2212         mrqc = 0;
2213         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2214         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2215         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2216         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2217         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2218         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2219         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2220         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2221         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2222
2223         /* If the mapping doesn't fit any supported, return */
2224         if (mrqc == 0)
2225                 return -EINVAL;
2226
2227         if (key != NULL)
2228                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2229                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2230
2231         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2232
2233         return 0;
2234 }
2235
2236 static int
2237 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2238         struct rte_eth_rss_conf *rss_conf)
2239 {
2240         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2241         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2242         uint32_t mrqc;
2243         uint64_t hf;
2244         int i;
2245
2246         PMD_INIT_FUNC_TRACE();
2247
2248         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2249                                 FM10K_RSSRK_ENTRIES_PER_REG))
2250                 return -EINVAL;
2251
2252         if (key != NULL)
2253                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2254                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2255
2256         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2257         hf = 0;
2258         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2259         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2260         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2261         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2262         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2263         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2264         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2265         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2266         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2267
2268         rss_conf->rss_hf = hf;
2269
2270         return 0;
2271 }
2272
2273 static void
2274 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2275 {
2276         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2277         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2278
2279         /* Bind all local non-queue interrupt to vector 0 */
2280         int_map |= FM10K_MISC_VEC_ID;
2281
2282         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2283         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2284         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2285         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2286         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2287         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2288
2289         /* Enable misc causes */
2290         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2291                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2292                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2293                                 FM10K_EIMR_ENABLE(MAILBOX) |
2294                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2295                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2296                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2297                                 FM10K_EIMR_ENABLE(VFLR));
2298
2299         /* Enable ITR 0 */
2300         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2301                                         FM10K_ITR_MASK_CLEAR);
2302         FM10K_WRITE_FLUSH(hw);
2303 }
2304
2305 static void
2306 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2307 {
2308         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2309         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2310
2311         int_map |= FM10K_MISC_VEC_ID;
2312
2313         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2314         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2315         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2316         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2317         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2318         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2319
2320         /* Disable misc causes */
2321         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2322                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2323                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2324                                 FM10K_EIMR_DISABLE(MAILBOX) |
2325                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2326                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2327                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2328                                 FM10K_EIMR_DISABLE(VFLR));
2329
2330         /* Disable ITR 0 */
2331         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2332         FM10K_WRITE_FLUSH(hw);
2333 }
2334
2335 static void
2336 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2337 {
2338         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2339         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2340
2341         /* Bind all local non-queue interrupt to vector 0 */
2342         int_map |= FM10K_MISC_VEC_ID;
2343
2344         /* Only INT 0 available, other 15 are reserved. */
2345         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2346
2347         /* Enable ITR 0 */
2348         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2349                                         FM10K_ITR_MASK_CLEAR);
2350         FM10K_WRITE_FLUSH(hw);
2351 }
2352
2353 static void
2354 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2355 {
2356         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2357         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2358
2359         int_map |= FM10K_MISC_VEC_ID;
2360
2361         /* Only INT 0 available, other 15 are reserved. */
2362         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2363
2364         /* Disable ITR 0 */
2365         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2366         FM10K_WRITE_FLUSH(hw);
2367 }
2368
2369 static int
2370 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2371 {
2372         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2373         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2374
2375         /* Enable ITR */
2376         if (hw->mac.type == fm10k_mac_pf)
2377                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2378                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2379         else
2380                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2381                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2382         rte_intr_ack(&pdev->intr_handle);
2383         return 0;
2384 }
2385
2386 static int
2387 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2388 {
2389         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2390         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2391
2392         /* Disable ITR */
2393         if (hw->mac.type == fm10k_mac_pf)
2394                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2395                         FM10K_ITR_MASK_SET);
2396         else
2397                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2398                         FM10K_ITR_MASK_SET);
2399         return 0;
2400 }
2401
2402 static int
2403 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2404 {
2405         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2406         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2407         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2408         uint32_t intr_vector, vec;
2409         uint16_t queue_id;
2410         int result = 0;
2411
2412         /* fm10k needs one separate interrupt for mailbox,
2413          * so only drivers which support multiple interrupt vectors
2414          * e.g. vfio-pci can work for fm10k interrupt mode
2415          */
2416         if (!rte_intr_cap_multiple(intr_handle) ||
2417                         dev->data->dev_conf.intr_conf.rxq == 0)
2418                 return result;
2419
2420         intr_vector = dev->data->nb_rx_queues;
2421
2422         /* disable interrupt first */
2423         rte_intr_disable(intr_handle);
2424         if (hw->mac.type == fm10k_mac_pf)
2425                 fm10k_dev_disable_intr_pf(dev);
2426         else
2427                 fm10k_dev_disable_intr_vf(dev);
2428
2429         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2430                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2431                 result = -EIO;
2432         }
2433
2434         if (rte_intr_dp_is_en(intr_handle) && !result) {
2435                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2436                         dev->data->nb_rx_queues * sizeof(int), 0);
2437                 if (intr_handle->intr_vec) {
2438                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2439                                         queue_id < dev->data->nb_rx_queues;
2440                                         queue_id++) {
2441                                 intr_handle->intr_vec[queue_id] = vec;
2442                                 if (vec < intr_handle->nb_efd - 1
2443                                                 + FM10K_RX_VEC_START)
2444                                         vec++;
2445                         }
2446                 } else {
2447                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2448                                 " intr_vec", dev->data->nb_rx_queues);
2449                         rte_intr_efd_disable(intr_handle);
2450                         result = -ENOMEM;
2451                 }
2452         }
2453
2454         if (hw->mac.type == fm10k_mac_pf)
2455                 fm10k_dev_enable_intr_pf(dev);
2456         else
2457                 fm10k_dev_enable_intr_vf(dev);
2458         rte_intr_enable(intr_handle);
2459         hw->mac.ops.update_int_moderator(hw);
2460         return result;
2461 }
2462
2463 static int
2464 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2465 {
2466         struct fm10k_fault fault;
2467         int err;
2468         const char *estr = "Unknown error";
2469
2470         /* Process PCA fault */
2471         if (eicr & FM10K_EICR_PCA_FAULT) {
2472                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2473                 if (err)
2474                         goto error;
2475                 switch (fault.type) {
2476                 case PCA_NO_FAULT:
2477                         estr = "PCA_NO_FAULT"; break;
2478                 case PCA_UNMAPPED_ADDR:
2479                         estr = "PCA_UNMAPPED_ADDR"; break;
2480                 case PCA_BAD_QACCESS_PF:
2481                         estr = "PCA_BAD_QACCESS_PF"; break;
2482                 case PCA_BAD_QACCESS_VF:
2483                         estr = "PCA_BAD_QACCESS_VF"; break;
2484                 case PCA_MALICIOUS_REQ:
2485                         estr = "PCA_MALICIOUS_REQ"; break;
2486                 case PCA_POISONED_TLP:
2487                         estr = "PCA_POISONED_TLP"; break;
2488                 case PCA_TLP_ABORT:
2489                         estr = "PCA_TLP_ABORT"; break;
2490                 default:
2491                         goto error;
2492                 }
2493                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2494                         estr, fault.func ? "VF" : "PF", fault.func,
2495                         fault.address, fault.specinfo);
2496         }
2497
2498         /* Process THI fault */
2499         if (eicr & FM10K_EICR_THI_FAULT) {
2500                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2501                 if (err)
2502                         goto error;
2503                 switch (fault.type) {
2504                 case THI_NO_FAULT:
2505                         estr = "THI_NO_FAULT"; break;
2506                 case THI_MAL_DIS_Q_FAULT:
2507                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2508                 default:
2509                         goto error;
2510                 }
2511                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2512                         estr, fault.func ? "VF" : "PF", fault.func,
2513                         fault.address, fault.specinfo);
2514         }
2515
2516         /* Process FUM fault */
2517         if (eicr & FM10K_EICR_FUM_FAULT) {
2518                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2519                 if (err)
2520                         goto error;
2521                 switch (fault.type) {
2522                 case FUM_NO_FAULT:
2523                         estr = "FUM_NO_FAULT"; break;
2524                 case FUM_UNMAPPED_ADDR:
2525                         estr = "FUM_UNMAPPED_ADDR"; break;
2526                 case FUM_POISONED_TLP:
2527                         estr = "FUM_POISONED_TLP"; break;
2528                 case FUM_BAD_VF_QACCESS:
2529                         estr = "FUM_BAD_VF_QACCESS"; break;
2530                 case FUM_ADD_DECODE_ERR:
2531                         estr = "FUM_ADD_DECODE_ERR"; break;
2532                 case FUM_RO_ERROR:
2533                         estr = "FUM_RO_ERROR"; break;
2534                 case FUM_QPRC_CRC_ERROR:
2535                         estr = "FUM_QPRC_CRC_ERROR"; break;
2536                 case FUM_CSR_TIMEOUT:
2537                         estr = "FUM_CSR_TIMEOUT"; break;
2538                 case FUM_INVALID_TYPE:
2539                         estr = "FUM_INVALID_TYPE"; break;
2540                 case FUM_INVALID_LENGTH:
2541                         estr = "FUM_INVALID_LENGTH"; break;
2542                 case FUM_INVALID_BE:
2543                         estr = "FUM_INVALID_BE"; break;
2544                 case FUM_INVALID_ALIGN:
2545                         estr = "FUM_INVALID_ALIGN"; break;
2546                 default:
2547                         goto error;
2548                 }
2549                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2550                         estr, fault.func ? "VF" : "PF", fault.func,
2551                         fault.address, fault.specinfo);
2552         }
2553
2554         return 0;
2555 error:
2556         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2557         return err;
2558 }
2559
2560 /**
2561  * PF interrupt handler triggered by NIC for handling specific interrupt.
2562  *
2563  * @param handle
2564  *  Pointer to interrupt handle.
2565  * @param param
2566  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2567  *
2568  * @return
2569  *  void
2570  */
2571 static void
2572 fm10k_dev_interrupt_handler_pf(void *param)
2573 {
2574         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2575         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2576         uint32_t cause, status;
2577         struct fm10k_dev_info *dev_info =
2578                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2579         int status_mbx;
2580         s32 err;
2581
2582         if (hw->mac.type != fm10k_mac_pf)
2583                 return;
2584
2585         cause = FM10K_READ_REG(hw, FM10K_EICR);
2586
2587         /* Handle PCI fault cases */
2588         if (cause & FM10K_EICR_FAULT_MASK) {
2589                 PMD_INIT_LOG(ERR, "INT: find fault!");
2590                 fm10k_dev_handle_fault(hw, cause);
2591         }
2592
2593         /* Handle switch up/down */
2594         if (cause & FM10K_EICR_SWITCHNOTREADY)
2595                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2596
2597         if (cause & FM10K_EICR_SWITCHREADY) {
2598                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2599                 if (dev_info->sm_down == 1) {
2600                         fm10k_mbx_lock(hw);
2601
2602                         /* For recreating logical ports */
2603                         status_mbx = hw->mac.ops.update_lport_state(hw,
2604                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2605                         if (status_mbx == FM10K_SUCCESS)
2606                                 PMD_INIT_LOG(INFO,
2607                                         "INT: Recreated Logical port");
2608                         else
2609                                 PMD_INIT_LOG(INFO,
2610                                         "INT: Logical ports weren't recreated");
2611
2612                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2613                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2614                         if (status_mbx != FM10K_SUCCESS)
2615                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2616
2617                         fm10k_mbx_unlock(hw);
2618
2619                         /* first clear the internal SW recording structure */
2620                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2621                                                 ETH_MQ_RX_VMDQ_FLAG))
2622                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2623                                         false);
2624
2625                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2626                                         MAIN_VSI_POOL_NUMBER);
2627
2628                         /*
2629                          * Add default mac address and vlan for the logical
2630                          * ports that have been created, leave to the
2631                          * application to fully recover Rx filtering.
2632                          */
2633                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2634                                         MAIN_VSI_POOL_NUMBER);
2635
2636                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2637                                                 ETH_MQ_RX_VMDQ_FLAG))
2638                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2639                                         true);
2640
2641                         dev_info->sm_down = 0;
2642                         _rte_eth_dev_callback_process(dev,
2643                                         RTE_ETH_EVENT_INTR_LSC,
2644                                         NULL);
2645                 }
2646         }
2647
2648         /* Handle mailbox message */
2649         fm10k_mbx_lock(hw);
2650         err = hw->mbx.ops.process(hw, &hw->mbx);
2651         fm10k_mbx_unlock(hw);
2652
2653         if (err == FM10K_ERR_RESET_REQUESTED) {
2654                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2655                 dev_info->sm_down = 1;
2656                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2657                                 NULL);
2658         }
2659
2660         /* Handle SRAM error */
2661         if (cause & FM10K_EICR_SRAMERROR) {
2662                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2663
2664                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2665                 /* Write to clear pending bits */
2666                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2667
2668                 /* Todo: print out error message after shared code  updates */
2669         }
2670
2671         /* Clear these 3 events if having any */
2672         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2673                  FM10K_EICR_SWITCHREADY;
2674         if (cause)
2675                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2676
2677         /* Re-enable interrupt from device side */
2678         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2679                                         FM10K_ITR_MASK_CLEAR);
2680         /* Re-enable interrupt from host side */
2681         rte_intr_ack(dev->intr_handle);
2682 }
2683
2684 /**
2685  * VF interrupt handler triggered by NIC for handling specific interrupt.
2686  *
2687  * @param handle
2688  *  Pointer to interrupt handle.
2689  * @param param
2690  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2691  *
2692  * @return
2693  *  void
2694  */
2695 static void
2696 fm10k_dev_interrupt_handler_vf(void *param)
2697 {
2698         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2699         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2700         struct fm10k_mbx_info *mbx = &hw->mbx;
2701         struct fm10k_dev_info *dev_info =
2702                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2703         const enum fm10k_mbx_state state = mbx->state;
2704         int status_mbx;
2705
2706         if (hw->mac.type != fm10k_mac_vf)
2707                 return;
2708
2709         /* Handle mailbox message if lock is acquired */
2710         fm10k_mbx_lock(hw);
2711         hw->mbx.ops.process(hw, &hw->mbx);
2712         fm10k_mbx_unlock(hw);
2713
2714         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2715                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2716
2717                 fm10k_mbx_lock(hw);
2718                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2719                                 MAX_LPORT_NUM, 1);
2720                 fm10k_mbx_unlock(hw);
2721
2722                 /* Setting reset flag */
2723                 dev_info->sm_down = 1;
2724                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2725                                 NULL);
2726         }
2727
2728         if (dev_info->sm_down == 1 &&
2729                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2730                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2731                 fm10k_mbx_lock(hw);
2732                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2733                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2734                 if (status_mbx != FM10K_SUCCESS)
2735                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2736                 fm10k_mbx_unlock(hw);
2737
2738                 /* first clear the internal SW recording structure */
2739                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2740                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2741                                 MAIN_VSI_POOL_NUMBER);
2742
2743                 /*
2744                  * Add default mac address and vlan for the logical ports that
2745                  * have been created, leave to the application to fully recover
2746                  * Rx filtering.
2747                  */
2748                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2749                                 MAIN_VSI_POOL_NUMBER);
2750                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2751
2752                 dev_info->sm_down = 0;
2753                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2754                                 NULL);
2755         }
2756
2757         /* Re-enable interrupt from device side */
2758         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2759                                         FM10K_ITR_MASK_CLEAR);
2760         /* Re-enable interrupt from host side */
2761         rte_intr_ack(dev->intr_handle);
2762 }
2763
2764 /* Mailbox message handler in VF */
2765 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2766         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2767         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2768         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2769         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2770 };
2771
2772 static int
2773 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2774 {
2775         int err = 0;
2776
2777         /* Initialize mailbox lock */
2778         fm10k_mbx_initlock(hw);
2779
2780         /* Replace default message handler with new ones */
2781         if (hw->mac.type == fm10k_mac_vf)
2782                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2783
2784         if (err) {
2785                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2786                                 err);
2787                 return err;
2788         }
2789         /* Connect to SM for PF device or PF for VF device */
2790         return hw->mbx.ops.connect(hw, &hw->mbx);
2791 }
2792
2793 static void
2794 fm10k_close_mbx_service(struct fm10k_hw *hw)
2795 {
2796         /* Disconnect from SM for PF device or PF for VF device */
2797         hw->mbx.ops.disconnect(hw, &hw->mbx);
2798 }
2799
2800 static void
2801 fm10k_dev_close(struct rte_eth_dev *dev)
2802 {
2803         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2804         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2805         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2806
2807         PMD_INIT_FUNC_TRACE();
2808
2809         fm10k_mbx_lock(hw);
2810         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2811                 MAX_LPORT_NUM, false);
2812         fm10k_mbx_unlock(hw);
2813
2814         /* allow 100ms for device to quiesce */
2815         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
2816
2817         /* Stop mailbox service first */
2818         fm10k_close_mbx_service(hw);
2819         fm10k_dev_stop(dev);
2820         fm10k_dev_queue_release(dev);
2821         fm10k_stop_hw(hw);
2822
2823         dev->dev_ops = NULL;
2824         dev->rx_pkt_burst = NULL;
2825         dev->tx_pkt_burst = NULL;
2826
2827         /* disable uio/vfio intr */
2828         rte_intr_disable(intr_handle);
2829
2830         /*PF/VF has different interrupt handling mechanism */
2831         if (hw->mac.type == fm10k_mac_pf) {
2832                 /* disable interrupt */
2833                 fm10k_dev_disable_intr_pf(dev);
2834
2835                 /* unregister callback func to eal lib */
2836                 rte_intr_callback_unregister(intr_handle,
2837                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2838         } else {
2839                 /* disable interrupt */
2840                 fm10k_dev_disable_intr_vf(dev);
2841
2842                 rte_intr_callback_unregister(intr_handle,
2843                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2844         }
2845 }
2846
2847 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2848         .dev_configure          = fm10k_dev_configure,
2849         .dev_start              = fm10k_dev_start,
2850         .dev_stop               = fm10k_dev_stop,
2851         .dev_close              = fm10k_dev_close,
2852         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2853         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2854         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2855         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2856         .stats_get              = fm10k_stats_get,
2857         .xstats_get             = fm10k_xstats_get,
2858         .xstats_get_names       = fm10k_xstats_get_names,
2859         .stats_reset            = fm10k_stats_reset,
2860         .xstats_reset           = fm10k_stats_reset,
2861         .link_update            = fm10k_link_update,
2862         .dev_infos_get          = fm10k_dev_infos_get,
2863         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2864         .vlan_filter_set        = fm10k_vlan_filter_set,
2865         .vlan_offload_set       = fm10k_vlan_offload_set,
2866         .mac_addr_add           = fm10k_macaddr_add,
2867         .mac_addr_remove        = fm10k_macaddr_remove,
2868         .rx_queue_start         = fm10k_dev_rx_queue_start,
2869         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2870         .tx_queue_start         = fm10k_dev_tx_queue_start,
2871         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2872         .rx_queue_setup         = fm10k_rx_queue_setup,
2873         .rx_queue_release       = fm10k_rx_queue_release,
2874         .tx_queue_setup         = fm10k_tx_queue_setup,
2875         .tx_queue_release       = fm10k_tx_queue_release,
2876         .rx_queue_count         = fm10k_dev_rx_queue_count,
2877         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2878         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2879         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2880         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2881         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2882         .reta_update            = fm10k_reta_update,
2883         .reta_query             = fm10k_reta_query,
2884         .rss_hash_update        = fm10k_rss_hash_update,
2885         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2886 };
2887
2888 static int ftag_check_handler(__rte_unused const char *key,
2889                 const char *value, __rte_unused void *opaque)
2890 {
2891         if (strcmp(value, "1"))
2892                 return -1;
2893
2894         return 0;
2895 }
2896
2897 static int
2898 fm10k_check_ftag(struct rte_devargs *devargs)
2899 {
2900         struct rte_kvargs *kvlist;
2901         const char *ftag_key = "enable_ftag";
2902
2903         if (devargs == NULL)
2904                 return 0;
2905
2906         kvlist = rte_kvargs_parse(devargs->args, NULL);
2907         if (kvlist == NULL)
2908                 return 0;
2909
2910         if (!rte_kvargs_count(kvlist, ftag_key)) {
2911                 rte_kvargs_free(kvlist);
2912                 return 0;
2913         }
2914         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2915         if (rte_kvargs_process(kvlist, ftag_key,
2916                                 ftag_check_handler, NULL) < 0) {
2917                 rte_kvargs_free(kvlist);
2918                 return 0;
2919         }
2920         rte_kvargs_free(kvlist);
2921
2922         return 1;
2923 }
2924
2925 static uint16_t
2926 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2927                     uint16_t nb_pkts)
2928 {
2929         uint16_t nb_tx = 0;
2930         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2931
2932         while (nb_pkts) {
2933                 uint16_t ret, num;
2934
2935                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2936                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2937                                                  num);
2938                 nb_tx += ret;
2939                 nb_pkts -= ret;
2940                 if (ret < num)
2941                         break;
2942         }
2943
2944         return nb_tx;
2945 }
2946
2947 static void __attribute__((cold))
2948 fm10k_set_tx_function(struct rte_eth_dev *dev)
2949 {
2950         struct fm10k_tx_queue *txq;
2951         int i;
2952         int use_sse = 1;
2953         uint16_t tx_ftag_en = 0;
2954
2955         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2956                 /* primary process has set the ftag flag and offloads */
2957                 txq = dev->data->tx_queues[0];
2958                 if (fm10k_tx_vec_condition_check(txq)) {
2959                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2960                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2961                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2962                 } else {
2963                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2964                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2965                         dev->tx_pkt_prepare = NULL;
2966                 }
2967                 return;
2968         }
2969
2970         if (fm10k_check_ftag(dev->device->devargs))
2971                 tx_ftag_en = 1;
2972
2973         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2974                 txq = dev->data->tx_queues[i];
2975                 txq->tx_ftag_en = tx_ftag_en;
2976                 /* Check if Vector Tx is satisfied */
2977                 if (fm10k_tx_vec_condition_check(txq))
2978                         use_sse = 0;
2979         }
2980
2981         if (use_sse) {
2982                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2983                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2984                         txq = dev->data->tx_queues[i];
2985                         fm10k_txq_vec_setup(txq);
2986                 }
2987                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2988                 dev->tx_pkt_prepare = NULL;
2989         } else {
2990                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2991                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2992                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2993         }
2994 }
2995
2996 static void __attribute__((cold))
2997 fm10k_set_rx_function(struct rte_eth_dev *dev)
2998 {
2999         struct fm10k_dev_info *dev_info =
3000                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3001         uint16_t i, rx_using_sse;
3002         uint16_t rx_ftag_en = 0;
3003
3004         if (fm10k_check_ftag(dev->device->devargs))
3005                 rx_ftag_en = 1;
3006
3007         /* In order to allow Vector Rx there are a few configuration
3008          * conditions to be met.
3009          */
3010         if (!fm10k_rx_vec_condition_check(dev) &&
3011                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3012                 if (dev->data->scattered_rx)
3013                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3014                 else
3015                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3016         } else if (dev->data->scattered_rx)
3017                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3018         else
3019                 dev->rx_pkt_burst = fm10k_recv_pkts;
3020
3021         rx_using_sse =
3022                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3023                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3024
3025         if (rx_using_sse)
3026                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3027         else
3028                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3029
3030         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3031                 return;
3032
3033         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3034                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3035
3036                 rxq->rx_using_sse = rx_using_sse;
3037                 rxq->rx_ftag_en = rx_ftag_en;
3038         }
3039 }
3040
3041 static void
3042 fm10k_params_init(struct rte_eth_dev *dev)
3043 {
3044         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3045         struct fm10k_dev_info *info =
3046                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3047
3048         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3049          * there is no way to get link status without reading BAR4.  Until this
3050          * works, assume we have maximum bandwidth.
3051          * @todo - fix bus info
3052          */
3053         hw->bus_caps.speed = fm10k_bus_speed_8000;
3054         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3055         hw->bus_caps.payload = fm10k_bus_payload_512;
3056         hw->bus.speed = fm10k_bus_speed_8000;
3057         hw->bus.width = fm10k_bus_width_pcie_x8;
3058         hw->bus.payload = fm10k_bus_payload_256;
3059
3060         info->rx_vec_allowed = true;
3061         info->sm_down = false;
3062 }
3063
3064 static int
3065 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3066 {
3067         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3068         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3069         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3070         int diag, i;
3071         struct fm10k_macvlan_filter_info *macvlan;
3072
3073         PMD_INIT_FUNC_TRACE();
3074
3075         dev->dev_ops = &fm10k_eth_dev_ops;
3076         dev->rx_pkt_burst = &fm10k_recv_pkts;
3077         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3078         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3079
3080         /*
3081          * Primary process does the whole initialization, for secondary
3082          * processes, we just select the same Rx and Tx function as primary.
3083          */
3084         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3085                 fm10k_set_rx_function(dev);
3086                 fm10k_set_tx_function(dev);
3087                 return 0;
3088         }
3089
3090         rte_eth_copy_pci_info(dev, pdev);
3091
3092         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3093         memset(macvlan, 0, sizeof(*macvlan));
3094         /* Vendor and Device ID need to be set before init of shared code */
3095         memset(hw, 0, sizeof(*hw));
3096         hw->device_id = pdev->id.device_id;
3097         hw->vendor_id = pdev->id.vendor_id;
3098         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3099         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3100         hw->revision_id = 0;
3101         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3102         if (hw->hw_addr == NULL) {
3103                 PMD_INIT_LOG(ERR, "Bad mem resource."
3104                         " Try to blacklist unused devices.");
3105                 return -EIO;
3106         }
3107
3108         /* Store fm10k_adapter pointer */
3109         hw->back = dev->data->dev_private;
3110
3111         /* Initialize the shared code */
3112         diag = fm10k_init_shared_code(hw);
3113         if (diag != FM10K_SUCCESS) {
3114                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3115                 return -EIO;
3116         }
3117
3118         /* Initialize parameters */
3119         fm10k_params_init(dev);
3120
3121         /* Initialize the hw */
3122         diag = fm10k_init_hw(hw);
3123         if (diag != FM10K_SUCCESS) {
3124                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3125                 return -EIO;
3126         }
3127
3128         /* Initialize MAC address(es) */
3129         dev->data->mac_addrs = rte_zmalloc("fm10k",
3130                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3131         if (dev->data->mac_addrs == NULL) {
3132                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3133                 return -ENOMEM;
3134         }
3135
3136         diag = fm10k_read_mac_addr(hw);
3137
3138         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3139                         &dev->data->mac_addrs[0]);
3140
3141         if (diag != FM10K_SUCCESS ||
3142                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3143
3144                 /* Generate a random addr */
3145                 rte_eth_random_addr(hw->mac.addr);
3146                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3147                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3148                 &dev->data->mac_addrs[0]);
3149         }
3150
3151         /* Pass the information to the rte_eth_dev_close() that it should also
3152          * release the private port resources.
3153          */
3154         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
3155
3156         /* Reset the hw statistics */
3157         diag = fm10k_stats_reset(dev);
3158         if (diag != 0) {
3159                 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag);
3160                 return diag;
3161         }
3162
3163         /* Reset the hw */
3164         diag = fm10k_reset_hw(hw);
3165         if (diag != FM10K_SUCCESS) {
3166                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3167                 return -EIO;
3168         }
3169
3170         /* Setup mailbox service */
3171         diag = fm10k_setup_mbx_service(hw);
3172         if (diag != FM10K_SUCCESS) {
3173                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3174                 return -EIO;
3175         }
3176
3177         /*PF/VF has different interrupt handling mechanism */
3178         if (hw->mac.type == fm10k_mac_pf) {
3179                 /* register callback func to eal lib */
3180                 rte_intr_callback_register(intr_handle,
3181                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3182
3183                 /* enable MISC interrupt */
3184                 fm10k_dev_enable_intr_pf(dev);
3185         } else { /* VF */
3186                 rte_intr_callback_register(intr_handle,
3187                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3188
3189                 fm10k_dev_enable_intr_vf(dev);
3190         }
3191
3192         /* Enable intr after callback registered */
3193         rte_intr_enable(intr_handle);
3194
3195         hw->mac.ops.update_int_moderator(hw);
3196
3197         /* Make sure Switch Manager is ready before going forward. */
3198         if (hw->mac.type == fm10k_mac_pf) {
3199                 int switch_ready = 0;
3200
3201                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3202                         fm10k_mbx_lock(hw);
3203                         hw->mac.ops.get_host_state(hw, &switch_ready);
3204                         fm10k_mbx_unlock(hw);
3205                         if (switch_ready)
3206                                 break;
3207                         /* Delay some time to acquire async LPORT_MAP info. */
3208                         rte_delay_us(WAIT_SWITCH_MSG_US);
3209                 }
3210
3211                 if (switch_ready == 0) {
3212                         PMD_INIT_LOG(ERR, "switch is not ready");
3213                         return -1;
3214                 }
3215         }
3216
3217         /*
3218          * Below function will trigger operations on mailbox, acquire lock to
3219          * avoid race condition from interrupt handler. Operations on mailbox
3220          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3221          * will handle and generate an interrupt to our side. Then,  FIFO in
3222          * mailbox will be touched.
3223          */
3224         fm10k_mbx_lock(hw);
3225         /* Enable port first */
3226         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3227                                         MAX_LPORT_NUM, 1);
3228
3229         /* Set unicast mode by default. App can change to other mode in other
3230          * API func.
3231          */
3232         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3233                                         FM10K_XCAST_MODE_NONE);
3234
3235         fm10k_mbx_unlock(hw);
3236
3237         /* Make sure default VID is ready before going forward. */
3238         if (hw->mac.type == fm10k_mac_pf) {
3239                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3240                         if (hw->mac.default_vid)
3241                                 break;
3242                         /* Delay some time to acquire async port VLAN info. */
3243                         rte_delay_us(WAIT_SWITCH_MSG_US);
3244                 }
3245
3246                 if (!hw->mac.default_vid) {
3247                         PMD_INIT_LOG(ERR, "default VID is not ready");
3248                         return -1;
3249                 }
3250         }
3251
3252         /* Add default mac address */
3253         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3254                 MAIN_VSI_POOL_NUMBER);
3255
3256         return 0;
3257 }
3258
3259 static int
3260 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3261 {
3262         PMD_INIT_FUNC_TRACE();
3263
3264         /* only uninitialize in the primary process */
3265         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3266                 return 0;
3267
3268         /* safe to close dev here */
3269         fm10k_dev_close(dev);
3270
3271         return 0;
3272 }
3273
3274 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3275         struct rte_pci_device *pci_dev)
3276 {
3277         return rte_eth_dev_pci_generic_probe(pci_dev,
3278                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3279 }
3280
3281 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3282 {
3283         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3284 }
3285
3286 /*
3287  * The set of PCI devices this driver supports. This driver will enable both PF
3288  * and SRIOV-VF devices.
3289  */
3290 static const struct rte_pci_id pci_id_fm10k_map[] = {
3291         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3292         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3293         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3294         { .vendor_id = 0, /* sentinel */ },
3295 };
3296
3297 static struct rte_pci_driver rte_pmd_fm10k = {
3298         .id_table = pci_id_fm10k_map,
3299         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3300         .probe = eth_fm10k_pci_probe,
3301         .remove = eth_fm10k_pci_remove,
3302 };
3303
3304 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3305 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3306 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3307
3308 RTE_INIT(fm10k_init_log)
3309 {
3310         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3311         if (fm10k_logtype_init >= 0)
3312                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3313         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3314         if (fm10k_logtype_driver >= 0)
3315                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3316 }