drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 __rte_weak int
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 __rte_weak uint16_t
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 __rte_weak uint16_t
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 __rte_weak int
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 __rte_weak void
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 __rte_weak void
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 __rte_weak int
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 __rte_weak uint16_t
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         /* multipe queue mode checking */
 455         ret  = fm10k_check_mq_mode(dev);
 456         if (ret != 0) {
 457                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 458                             ret);
 459                 return ret;
 460         }
 461
 462         dev->data->scattered_rx = 0;
 463
 464         return 0;
 465 }
 466
 467 static void
 468 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 469 {
 470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 471         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 472         uint32_t i;
 473
 474         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 475
 476         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 477                 if (!vmdq_conf->pool_map[i].pools)
 478                         continue;
 479                 fm10k_mbx_lock(hw);
 480                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 481                 fm10k_mbx_unlock(hw);
 482         }
 483 }
 484
 485 static void
 486 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 487 {
 488         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 489
 490         /* Add default mac address */
 491         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 492                 MAIN_VSI_POOL_NUMBER);
 493 }
 494
 495 static void
 496 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 497 {
 498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 499         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 500         uint32_t mrqc, *key, i, reta, j;
 501         uint64_t hf;
 502
 503 #define RSS_KEY_SIZE 40
 504         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 505                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 506                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 507                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 508                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 509                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 510         };
 511
 512         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 513                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 514                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 515                 return;
 516         }
 517
 518         /* random key is rss_intel_key (default) or user provided (rss_key) */
 519         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 520                 key = (uint32_t *)rss_intel_key;
 521         else
 522                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 523
 524         /* Now fill our hash function seeds, 4 bytes at a time */
 525         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 526                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 527
 528         /*
 529          * Fill in redirection table
 530          * The byte-swap is needed because NIC registers are in
 531          * little-endian order.
 532          */
 533         reta = 0;
 534         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 535                 if (j == dev->data->nb_rx_queues)
 536                         j = 0;
 537                 reta = (reta << CHAR_BIT) | j;
 538                 if ((i & 3) == 3)
 539                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 540                                         rte_bswap32(reta));
 541         }
 542
 543         /*
 544          * Generate RSS hash based on packet types, TCP/UDP
 545          * port numbers and/or IPv4/v6 src and dst addresses
 546          */
 547         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 548         mrqc = 0;
 549         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 550         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 551         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 552         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 553         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 554         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 555         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 556         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 557         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 558
 559         if (mrqc == 0) {
 560                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 561                         "supported", hf);
 562                 return;
 563         }
 564
 565         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 566 }
 567
 568 static void
 569 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 570 {
 571         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 572         uint32_t i;
 573
 574         for (i = 0; i < nb_lport_new; i++) {
 575                 /* Set unicast mode by default. App can change
 576                  * to other mode in other API func.
 577                  */
 578                 fm10k_mbx_lock(hw);
 579                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 580                         FM10K_XCAST_MODE_NONE);
 581                 fm10k_mbx_unlock(hw);
 582         }
 583 }
 584
 585 static void
 586 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 587 {
 588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 589         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 590         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 591         struct fm10k_macvlan_filter_info *macvlan;
 592         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 593         uint16_t nb_lport_new;
 594
 595         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 596         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 597
 598         fm10k_dev_rss_configure(dev);
 599
 600         /* only PF supports VMDQ */
 601         if (hw->mac.type != fm10k_mac_pf)
 602                 return;
 603
 604         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 605                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 606
 607         /* no pool number change, no need to update logic port and VLAN/MAC */
 608         if (macvlan->nb_queue_pools == nb_queue_pools)
 609                 return;
 610
 611         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 612         fm10k_dev_logic_port_update(dev, nb_lport_new);
 613
 614         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 615         memset(dev->data->mac_addrs, 0,
 616                 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 617         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
 618                 &dev->data->mac_addrs[0]);
 619         memset(macvlan, 0, sizeof(*macvlan));
 620         macvlan->nb_queue_pools = nb_queue_pools;
 621
 622         if (nb_queue_pools)
 623                 fm10k_dev_vmdq_rx_configure(dev);
 624         else
 625                 fm10k_dev_pf_main_vsi_reset(dev);
 626 }
 627
 628 static int
 629 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int i, ret;
 633         struct fm10k_tx_queue *txq;
 634         uint64_t base_addr;
 635         uint32_t size;
 636
 637         /* Disable TXINT to avoid possible interrupt */
 638         for (i = 0; i < hw->mac.max_queues; i++)
 639                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 640                                 3 << FM10K_TXINT_TIMER_SHIFT);
 641
 642         /* Setup TX queue */
 643         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 644                 txq = dev->data->tx_queues[i];
 645                 base_addr = txq->hw_ring_phys_addr;
 646                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 647
 648                 /* disable queue to avoid issues while updating state */
 649                 ret = tx_queue_disable(hw, i);
 650                 if (ret) {
 651                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 652                         return -1;
 653                 }
 654                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 655                  * register is read-only for VF.
 656                  */
 657                 if (fm10k_check_ftag(dev->device->devargs)) {
 658                         if (hw->mac.type == fm10k_mac_pf) {
 659                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 660                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 661                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 662                         } else {
 663                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 664                                 return -ENOTSUP;
 665                         }
 666                 }
 667
 668                 /* set location and size for descriptor ring */
 669                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 670                                 base_addr & UINT64_LOWER_32BITS_MASK);
 671                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 672                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 673                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 674
 675                 /* assign default SGLORT for each TX queue by PF */
 676                 if (hw->mac.type == fm10k_mac_pf)
 677                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 678         }
 679
 680         /* set up vector or scalar TX function as appropriate */
 681         fm10k_set_tx_function(dev);
 682
 683         return 0;
 684 }
 685
 686 static int
 687 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 688 {
 689         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 690         struct fm10k_macvlan_filter_info *macvlan;
 691         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 692         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 693         int i, ret;
 694         struct fm10k_rx_queue *rxq;
 695         uint64_t base_addr;
 696         uint32_t size;
 697         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 698         uint32_t logic_port = hw->mac.dglort_map;
 699         uint16_t buf_size;
 700         uint16_t queue_stride = 0;
 701
 702         /* enable RXINT for interrupt mode */
 703         i = 0;
 704         if (rte_intr_dp_is_en(intr_handle)) {
 705                 for (; i < dev->data->nb_rx_queues; i++) {
 706                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 707                         if (hw->mac.type == fm10k_mac_pf)
 708                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 709                                         FM10K_ITR_AUTOMASK |
 710                                         FM10K_ITR_MASK_CLEAR);
 711                         else
 712                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 713                                         FM10K_ITR_AUTOMASK |
 714                                         FM10K_ITR_MASK_CLEAR);
 715                 }
 716         }
 717         /* Disable other RXINT to avoid possible interrupt */
 718         for (; i < hw->mac.max_queues; i++)
 719                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 720                         3 << FM10K_RXINT_TIMER_SHIFT);
 721
 722         /* Setup RX queues */
 723         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 724                 rxq = dev->data->rx_queues[i];
 725                 base_addr = rxq->hw_ring_phys_addr;
 726                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 727
 728                 /* disable queue to avoid issues while updating state */
 729                 ret = rx_queue_disable(hw, i);
 730                 if (ret) {
 731                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 732                         return -1;
 733                 }
 734
 735                 /* Setup the Base and Length of the Rx Descriptor Ring */
 736                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 737                                 base_addr & UINT64_LOWER_32BITS_MASK);
 738                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 739                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 740                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 741
 742                 /* Configure the Rx buffer size for one buff without split */
 743                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 744                         RTE_PKTMBUF_HEADROOM);
 745                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 746                  * reserved for this purpose, and the worst case could be 511B.
 747                  * But SRR reg assumes all buffers have the same size. In order
 748                  * to fill the gap, we'll have to consider the worst case and
 749                  * assume 512B is reserved. If we don't do so, it's possible
 750                  * for HW to overwrite data to next mbuf.
 751                  */
 752                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 753
 754                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 755                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 756                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 757
 758                 /* It adds dual VLAN length for supporting dual VLAN */
 759                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 760                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 761                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 762                         uint32_t reg;
 763                         dev->data->scattered_rx = 1;
 764                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 765                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 766                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 767                 }
 768
 769                 /* Enable drop on empty, it's RO for VF */
 770                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 771                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 772
 773                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 774                 FM10K_WRITE_FLUSH(hw);
 775         }
 776
 777         /* Configure VMDQ/RSS if applicable */
 778         fm10k_dev_mq_rx_configure(dev);
 779
 780         /* Decide the best RX function */
 781         fm10k_set_rx_function(dev);
 782
 783         /* update RX_SGLORT for loopback suppress*/
 784         if (hw->mac.type != fm10k_mac_pf)
 785                 return 0;
 786         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 787         if (macvlan->nb_queue_pools)
 788                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 789         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 790                 if (i && queue_stride && !(i % queue_stride))
 791                         logic_port++;
 792                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 793         }
 794
 795         return 0;
 796 }
 797
 798 static int
 799 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802         int err;
 803         uint32_t reg;
 804         struct fm10k_rx_queue *rxq;
 805
 806         PMD_INIT_FUNC_TRACE();
 807
 808         rxq = dev->data->rx_queues[rx_queue_id];
 809         err = rx_queue_reset(rxq);
 810         if (err == -ENOMEM) {
 811                 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 812                 return err;
 813         } else if (err == -EINVAL) {
 814                 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 815                         " %d", err);
 816                 return err;
 817         }
 818
 819         /* Setup the HW Rx Head and Tail Descriptor Pointers
 820          * Note: this must be done AFTER the queue is enabled on real
 821          * hardware, but BEFORE the queue is enabled when using the
 822          * emulation platform. Do it in both places for now and remove
 823          * this comment and the following two register writes when the
 824          * emulation platform is no longer being used.
 825          */
 826         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 827         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 828
 829         /* Set PF ownership flag for PF devices */
 830         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 831         if (hw->mac.type == fm10k_mac_pf)
 832                 reg |= FM10K_RXQCTL_PF;
 833         reg |= FM10K_RXQCTL_ENABLE;
 834         /* enable RX queue */
 835         FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 836         FM10K_WRITE_FLUSH(hw);
 837
 838         /* Setup the HW Rx Head and Tail Descriptor Pointers
 839          * Note: this must be done AFTER the queue is enabled
 840          */
 841         FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 842         FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 843         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 844
 845         return 0;
 846 }
 847
 848 static int
 849 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 850 {
 851         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Disable RX queue */
 856         rx_queue_disable(hw, rx_queue_id);
 857
 858         /* Free mbuf and clean HW ring */
 859         rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 860         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 861
 862         return 0;
 863 }
 864
 865 static int
 866 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 867 {
 868         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 869         /** @todo - this should be defined in the shared code */
 870 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 871         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 872         struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 873
 874         PMD_INIT_FUNC_TRACE();
 875
 876         q->ops->reset(q);
 877
 878         /* reset head and tail pointers */
 879         FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 880         FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 881
 882         /* enable TX queue */
 883         FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 884                                 FM10K_TXDCTL_ENABLE | txdctl);
 885         FM10K_WRITE_FLUSH(hw);
 886         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 887
 888         return 0;
 889 }
 890
 891 static int
 892 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         tx_queue_disable(hw, tx_queue_id);
 899         tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 900         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 901
 902         return 0;
 903 }
 904
 905 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 906 {
 907         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 908                 != FM10K_DGLORTMAP_NONE);
 909 }
 910
 911 static void
 912 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 913 {
 914         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 915         int status;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         /* Return if it didn't acquire valid glort range */
 920         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 921                 return;
 922
 923         fm10k_mbx_lock(hw);
 924         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 925                                 FM10K_XCAST_MODE_PROMISC);
 926         fm10k_mbx_unlock(hw);
 927
 928         if (status != FM10K_SUCCESS)
 929                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 930 }
 931
 932 static void
 933 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 934 {
 935         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 936         uint8_t mode;
 937         int status;
 938
 939         PMD_INIT_FUNC_TRACE();
 940
 941         /* Return if it didn't acquire valid glort range */
 942         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 943                 return;
 944
 945         if (dev->data->all_multicast == 1)
 946                 mode = FM10K_XCAST_MODE_ALLMULTI;
 947         else
 948                 mode = FM10K_XCAST_MODE_NONE;
 949
 950         fm10k_mbx_lock(hw);
 951         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 952                                 mode);
 953         fm10k_mbx_unlock(hw);
 954
 955         if (status != FM10K_SUCCESS)
 956                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 957 }
 958
 959 static void
 960 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 961 {
 962         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 963         int status;
 964
 965         PMD_INIT_FUNC_TRACE();
 966
 967         /* Return if it didn't acquire valid glort range */
 968         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 969                 return;
 970
 971         /* If promiscuous mode is enabled, it doesn't make sense to enable
 972          * allmulticast and disable promiscuous since fm10k only can select
 973          * one of the modes.
 974          */
 975         if (dev->data->promiscuous) {
 976                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 977                         "needn't enable allmulticast");
 978                 return;
 979         }
 980
 981         fm10k_mbx_lock(hw);
 982         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 983                                 FM10K_XCAST_MODE_ALLMULTI);
 984         fm10k_mbx_unlock(hw);
 985
 986         if (status != FM10K_SUCCESS)
 987                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 988 }
 989
 990 static void
 991 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 992 {
 993         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 994         int status;
 995
 996         PMD_INIT_FUNC_TRACE();
 997
 998         /* Return if it didn't acquire valid glort range */
 999         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1000                 return;
1001
1002         if (dev->data->promiscuous) {
1003                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1004                         "since promisc mode is enabled");
1005                 return;
1006         }
1007
1008         fm10k_mbx_lock(hw);
1009         /* Change mode to unicast mode */
1010         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1011                                 FM10K_XCAST_MODE_NONE);
1012         fm10k_mbx_unlock(hw);
1013
1014         if (status != FM10K_SUCCESS)
1015                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1016 }
1017
1018 static void
1019 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1020 {
1021         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1022         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1023         uint16_t nb_queue_pools;
1024         struct fm10k_macvlan_filter_info *macvlan;
1025
1026         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1027         nb_queue_pools = macvlan->nb_queue_pools;
1028         pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0;
1029         rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len;
1030
1031         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1032         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1033         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1034                         hw->mac.dglort_map;
1035         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1036         /* Configure VMDQ/RSS DGlort Decoder */
1037         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1038
1039         /* Flow Director configurations, only queue number is valid. */
1040         dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1);
1041         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1042                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1043         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1044         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1045
1046         /* Invalidate all other GLORT entries */
1047         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1048                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1049                                 FM10K_DGLORTMAP_NONE);
1050 }
1051
1052 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1053 static int
1054 fm10k_dev_start(struct rte_eth_dev *dev)
1055 {
1056         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1057         int i, diag;
1058
1059         PMD_INIT_FUNC_TRACE();
1060
1061         /* stop, init, then start the hw */
1062         diag = fm10k_stop_hw(hw);
1063         if (diag != FM10K_SUCCESS) {
1064                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1065                 return -EIO;
1066         }
1067
1068         diag = fm10k_init_hw(hw);
1069         if (diag != FM10K_SUCCESS) {
1070                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1071                 return -EIO;
1072         }
1073
1074         diag = fm10k_start_hw(hw);
1075         if (diag != FM10K_SUCCESS) {
1076                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1077                 return -EIO;
1078         }
1079
1080         diag = fm10k_dev_tx_init(dev);
1081         if (diag) {
1082                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1083                 return diag;
1084         }
1085
1086         if (fm10k_dev_rxq_interrupt_setup(dev))
1087                 return -EIO;
1088
1089         diag = fm10k_dev_rx_init(dev);
1090         if (diag) {
1091                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1092                 return diag;
1093         }
1094
1095         if (hw->mac.type == fm10k_mac_pf)
1096                 fm10k_dev_dglort_map_configure(dev);
1097
1098         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1099                 struct fm10k_rx_queue *rxq;
1100                 rxq = dev->data->rx_queues[i];
1101
1102                 if (rxq->rx_deferred_start)
1103                         continue;
1104                 diag = fm10k_dev_rx_queue_start(dev, i);
1105                 if (diag != 0) {
1106                         int j;
1107                         for (j = 0; j < i; ++j)
1108                                 rx_queue_clean(dev->data->rx_queues[j]);
1109                         return diag;
1110                 }
1111         }
1112
1113         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1114                 struct fm10k_tx_queue *txq;
1115                 txq = dev->data->tx_queues[i];
1116
1117                 if (txq->tx_deferred_start)
1118                         continue;
1119                 diag = fm10k_dev_tx_queue_start(dev, i);
1120                 if (diag != 0) {
1121                         int j;
1122                         for (j = 0; j < i; ++j)
1123                                 tx_queue_clean(dev->data->tx_queues[j]);
1124                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1125                                 rx_queue_clean(dev->data->rx_queues[j]);
1126                         return diag;
1127                 }
1128         }
1129
1130         /* Update default vlan when not in VMDQ mode */
1131         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1132                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1133
1134         fm10k_link_update(dev, 0);
1135
1136         return 0;
1137 }
1138
1139 static void
1140 fm10k_dev_stop(struct rte_eth_dev *dev)
1141 {
1142         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1143         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1144         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1145         int i;
1146
1147         PMD_INIT_FUNC_TRACE();
1148
1149         if (dev->data->tx_queues)
1150                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1151                         fm10k_dev_tx_queue_stop(dev, i);
1152
1153         if (dev->data->rx_queues)
1154                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1155                         fm10k_dev_rx_queue_stop(dev, i);
1156
1157         /* Disable datapath event */
1158         if (rte_intr_dp_is_en(intr_handle)) {
1159                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1160                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1161                                 3 << FM10K_RXINT_TIMER_SHIFT);
1162                         if (hw->mac.type == fm10k_mac_pf)
1163                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1164                                         FM10K_ITR_MASK_SET);
1165                         else
1166                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1167                                         FM10K_ITR_MASK_SET);
1168                 }
1169         }
1170         /* Clean datapath event and queue/vec mapping */
1171         rte_intr_efd_disable(intr_handle);
1172         rte_free(intr_handle->intr_vec);
1173         intr_handle->intr_vec = NULL;
1174 }
1175
1176 static void
1177 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1178 {
1179         int i;
1180
1181         PMD_INIT_FUNC_TRACE();
1182
1183         if (dev->data->tx_queues) {
1184                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1185                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1186
1187                         tx_queue_free(txq);
1188                 }
1189         }
1190
1191         if (dev->data->rx_queues) {
1192                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1193                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1194         }
1195 }
1196
1197 static void
1198 fm10k_dev_close(struct rte_eth_dev *dev)
1199 {
1200         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1201
1202         PMD_INIT_FUNC_TRACE();
1203
1204         fm10k_mbx_lock(hw);
1205         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1206                 MAX_LPORT_NUM, false);
1207         fm10k_mbx_unlock(hw);
1208
1209         /* allow 100ms for device to quiesce */
1210         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1211
1212         /* Stop mailbox service first */
1213         fm10k_close_mbx_service(hw);
1214         fm10k_dev_stop(dev);
1215         fm10k_dev_queue_release(dev);
1216         fm10k_stop_hw(hw);
1217 }
1218
1219 static int
1220 fm10k_link_update(struct rte_eth_dev *dev,
1221         __rte_unused int wait_to_complete)
1222 {
1223         struct fm10k_dev_info *dev_info =
1224                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1225         PMD_INIT_FUNC_TRACE();
1226
1227         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1228         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1229         dev->data->dev_link.link_status =
1230                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1231         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1232
1233         return 0;
1234 }
1235
1236 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1237         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1238 {
1239         unsigned i, q;
1240         unsigned count = 0;
1241
1242         if (xstats_names != NULL) {
1243                 /* Note: limit checked in rte_eth_xstats_names() */
1244
1245                 /* Global stats */
1246                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1247                         snprintf(xstats_names[count].name,
1248                                 sizeof(xstats_names[count].name),
1249                                 "%s", fm10k_hw_stats_strings[count].name);
1250                         count++;
1251                 }
1252
1253                 /* PF queue stats */
1254                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1255                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1256                                 snprintf(xstats_names[count].name,
1257                                         sizeof(xstats_names[count].name),
1258                                         "rx_q%u_%s", q,
1259                                         fm10k_hw_stats_rx_q_strings[i].name);
1260                                 count++;
1261                         }
1262                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1263                                 snprintf(xstats_names[count].name,
1264                                         sizeof(xstats_names[count].name),
1265                                         "tx_q%u_%s", q,
1266                                         fm10k_hw_stats_tx_q_strings[i].name);
1267                                 count++;
1268                         }
1269                 }
1270         }
1271         return FM10K_NB_XSTATS;
1272 }
1273
1274 static int
1275 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1276                  unsigned n)
1277 {
1278         struct fm10k_hw_stats *hw_stats =
1279                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1280         unsigned i, q, count = 0;
1281
1282         if (n < FM10K_NB_XSTATS)
1283                 return FM10K_NB_XSTATS;
1284
1285         /* Global stats */
1286         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1287                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1288                         fm10k_hw_stats_strings[count].offset);
1289                 xstats[count].id = count;
1290                 count++;
1291         }
1292
1293         /* PF queue stats */
1294         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1295                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1296                         xstats[count].value =
1297                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1298                                 fm10k_hw_stats_rx_q_strings[i].offset);
1299                         xstats[count].id = count;
1300                         count++;
1301                 }
1302                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1303                         xstats[count].value =
1304                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1305                                 fm10k_hw_stats_tx_q_strings[i].offset);
1306                         xstats[count].id = count;
1307                         count++;
1308                 }
1309         }
1310
1311         return FM10K_NB_XSTATS;
1312 }
1313
1314 static int
1315 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1316 {
1317         uint64_t ipackets, opackets, ibytes, obytes, imissed;
1318         struct fm10k_hw *hw =
1319                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1320         struct fm10k_hw_stats *hw_stats =
1321                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1322         int i;
1323
1324         PMD_INIT_FUNC_TRACE();
1325
1326         fm10k_update_hw_stats(hw, hw_stats);
1327
1328         ipackets = opackets = ibytes = obytes = imissed = 0;
1329         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1330                 (i < hw->mac.max_queues); ++i) {
1331                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1332                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1333                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1334                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1335                 stats->q_errors[i]   = hw_stats->q[i].rx_drops.count;
1336                 ipackets += stats->q_ipackets[i];
1337                 opackets += stats->q_opackets[i];
1338                 ibytes   += stats->q_ibytes[i];
1339                 obytes   += stats->q_obytes[i];
1340                 imissed  += stats->q_errors[i];
1341         }
1342         stats->ipackets = ipackets;
1343         stats->opackets = opackets;
1344         stats->ibytes = ibytes;
1345         stats->obytes = obytes;
1346         stats->imissed = imissed;
1347         return 0;
1348 }
1349
1350 static void
1351 fm10k_stats_reset(struct rte_eth_dev *dev)
1352 {
1353         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1354         struct fm10k_hw_stats *hw_stats =
1355                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1356
1357         PMD_INIT_FUNC_TRACE();
1358
1359         memset(hw_stats, 0, sizeof(*hw_stats));
1360         fm10k_rebind_hw_stats(hw, hw_stats);
1361 }
1362
1363 static void
1364 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1365         struct rte_eth_dev_info *dev_info)
1366 {
1367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1369
1370         PMD_INIT_FUNC_TRACE();
1371
1372         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1373         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1374         dev_info->max_rx_queues      = hw->mac.max_queues;
1375         dev_info->max_tx_queues      = hw->mac.max_queues;
1376         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1377         dev_info->max_hash_mac_addrs = 0;
1378         dev_info->max_vfs            = pdev->max_vfs;
1379         dev_info->vmdq_pool_base     = 0;
1380         dev_info->vmdq_queue_base    = 0;
1381         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1382         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1383         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1384         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1385                                     dev_info->rx_queue_offload_capa;
1386         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1387         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1388                                     dev_info->tx_queue_offload_capa;
1389
1390         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1391         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1392         dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1393                                         ETH_RSS_IPV6 |
1394                                         ETH_RSS_IPV6_EX |
1395                                         ETH_RSS_NONFRAG_IPV4_TCP |
1396                                         ETH_RSS_NONFRAG_IPV6_TCP |
1397                                         ETH_RSS_IPV6_TCP_EX |
1398                                         ETH_RSS_NONFRAG_IPV4_UDP |
1399                                         ETH_RSS_NONFRAG_IPV6_UDP |
1400                                         ETH_RSS_IPV6_UDP_EX;
1401
1402         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1403                 .rx_thresh = {
1404                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1405                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1406                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1407                 },
1408                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1409                 .rx_drop_en = 0,
1410                 .offloads = 0,
1411         };
1412
1413         dev_info->default_txconf = (struct rte_eth_txconf) {
1414                 .tx_thresh = {
1415                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1416                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1417                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1418                 },
1419                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1420                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1421                 .offloads = 0,
1422         };
1423
1424         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1425                 .nb_max = FM10K_MAX_RX_DESC,
1426                 .nb_min = FM10K_MIN_RX_DESC,
1427                 .nb_align = FM10K_MULT_RX_DESC,
1428         };
1429
1430         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1431                 .nb_max = FM10K_MAX_TX_DESC,
1432                 .nb_min = FM10K_MIN_TX_DESC,
1433                 .nb_align = FM10K_MULT_TX_DESC,
1434                 .nb_seg_max = FM10K_TX_MAX_SEG,
1435                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1436         };
1437
1438         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1439                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1440                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1441 }
1442
1443 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1444 static const uint32_t *
1445 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1446 {
1447         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1448             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1449                 static uint32_t ptypes[] = {
1450                         /* refers to rx_desc_to_ol_flags() */
1451                         RTE_PTYPE_L2_ETHER,
1452                         RTE_PTYPE_L3_IPV4,
1453                         RTE_PTYPE_L3_IPV4_EXT,
1454                         RTE_PTYPE_L3_IPV6,
1455                         RTE_PTYPE_L3_IPV6_EXT,
1456                         RTE_PTYPE_L4_TCP,
1457                         RTE_PTYPE_L4_UDP,
1458                         RTE_PTYPE_UNKNOWN
1459                 };
1460
1461                 return ptypes;
1462         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1463                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1464                 static uint32_t ptypes_vec[] = {
1465                         /* refers to fm10k_desc_to_pktype_v() */
1466                         RTE_PTYPE_L3_IPV4,
1467                         RTE_PTYPE_L3_IPV4_EXT,
1468                         RTE_PTYPE_L3_IPV6,
1469                         RTE_PTYPE_L3_IPV6_EXT,
1470                         RTE_PTYPE_L4_TCP,
1471                         RTE_PTYPE_L4_UDP,
1472                         RTE_PTYPE_TUNNEL_GENEVE,
1473                         RTE_PTYPE_TUNNEL_NVGRE,
1474                         RTE_PTYPE_TUNNEL_VXLAN,
1475                         RTE_PTYPE_TUNNEL_GRE,
1476                         RTE_PTYPE_UNKNOWN
1477                 };
1478
1479                 return ptypes_vec;
1480         }
1481
1482         return NULL;
1483 }
1484 #else
1485 static const uint32_t *
1486 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1487 {
1488         return NULL;
1489 }
1490 #endif
1491
1492 static int
1493 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1494 {
1495         s32 result;
1496         uint16_t mac_num = 0;
1497         uint32_t vid_idx, vid_bit, mac_index;
1498         struct fm10k_hw *hw;
1499         struct fm10k_macvlan_filter_info *macvlan;
1500         struct rte_eth_dev_data *data = dev->data;
1501
1502         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1503         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1504
1505         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1506                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1507                 return -EINVAL;
1508         }
1509
1510         if (vlan_id > ETH_VLAN_ID_MAX) {
1511                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1512                 return -EINVAL;
1513         }
1514
1515         vid_idx = FM10K_VFTA_IDX(vlan_id);
1516         vid_bit = FM10K_VFTA_BIT(vlan_id);
1517         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1518         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1519                 return 0;
1520         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1521         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1522                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1523                         "in the VLAN filter table");
1524                 return -EINVAL;
1525         }
1526
1527         fm10k_mbx_lock(hw);
1528         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1529         fm10k_mbx_unlock(hw);
1530         if (result != FM10K_SUCCESS) {
1531                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1532                 return -EIO;
1533         }
1534
1535         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1536                         (result == FM10K_SUCCESS); mac_index++) {
1537                 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index]))
1538                         continue;
1539                 if (mac_num > macvlan->mac_num - 1) {
1540                         PMD_INIT_LOG(ERR, "MAC address number "
1541                                         "not match");
1542                         break;
1543                 }
1544                 fm10k_mbx_lock(hw);
1545                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1546                         data->mac_addrs[mac_index].addr_bytes,
1547                         vlan_id, on, 0);
1548                 fm10k_mbx_unlock(hw);
1549                 mac_num++;
1550         }
1551         if (result != FM10K_SUCCESS) {
1552                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1553                 return -EIO;
1554         }
1555
1556         if (on) {
1557                 macvlan->vlan_num++;
1558                 macvlan->vfta[vid_idx] |= vid_bit;
1559         } else {
1560                 macvlan->vlan_num--;
1561                 macvlan->vfta[vid_idx] &= ~vid_bit;
1562         }
1563         return 0;
1564 }
1565
1566 static int
1567 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1568 {
1569         if (mask & ETH_VLAN_STRIP_MASK) {
1570                 if (!(dev->data->dev_conf.rxmode.offloads &
1571                         DEV_RX_OFFLOAD_VLAN_STRIP))
1572                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1573                                         "always on in fm10k");
1574         }
1575
1576         if (mask & ETH_VLAN_EXTEND_MASK) {
1577                 if (dev->data->dev_conf.rxmode.offloads &
1578                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1579                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1580                                         "supported in fm10k");
1581         }
1582
1583         if (mask & ETH_VLAN_FILTER_MASK) {
1584                 if (!(dev->data->dev_conf.rxmode.offloads &
1585                         DEV_RX_OFFLOAD_VLAN_FILTER))
1586                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1587         }
1588
1589         return 0;
1590 }
1591
1592 /* Add/Remove a MAC address, and update filters to main VSI */
1593 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1594                 const u8 *mac, bool add, uint32_t pool)
1595 {
1596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1597         struct fm10k_macvlan_filter_info *macvlan;
1598         uint32_t i, j, k;
1599
1600         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1601
1602         if (pool != MAIN_VSI_POOL_NUMBER) {
1603                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1604                         "mac to pool %u", pool);
1605                 return;
1606         }
1607         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1608                 if (!macvlan->vfta[j])
1609                         continue;
1610                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1611                         if (!(macvlan->vfta[j] & (1 << k)))
1612                                 continue;
1613                         if (i + 1 > macvlan->vlan_num) {
1614                                 PMD_INIT_LOG(ERR, "vlan number not match");
1615                                 return;
1616                         }
1617                         fm10k_mbx_lock(hw);
1618                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1619                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1620                         fm10k_mbx_unlock(hw);
1621                         i++;
1622                 }
1623         }
1624 }
1625
1626 /* Add/Remove a MAC address, and update filters to VMDQ */
1627 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1628                 const u8 *mac, bool add, uint32_t pool)
1629 {
1630         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1631         struct fm10k_macvlan_filter_info *macvlan;
1632         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1633         uint32_t i;
1634
1635         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1636         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1637
1638         if (pool > macvlan->nb_queue_pools) {
1639                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1640                         " Max pool is %u",
1641                         pool, macvlan->nb_queue_pools);
1642                 return;
1643         }
1644         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1645                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1646                         continue;
1647                 fm10k_mbx_lock(hw);
1648                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1649                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1650                 fm10k_mbx_unlock(hw);
1651         }
1652 }
1653
1654 /* Add/Remove a MAC address, and update filters */
1655 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1656                 const u8 *mac, bool add, uint32_t pool)
1657 {
1658         struct fm10k_macvlan_filter_info *macvlan;
1659
1660         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1661
1662         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1663                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1664         else
1665                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1666
1667         if (add)
1668                 macvlan->mac_num++;
1669         else
1670                 macvlan->mac_num--;
1671 }
1672
1673 /* Add a MAC address, and update filters */
1674 static int
1675 fm10k_macaddr_add(struct rte_eth_dev *dev,
1676                 struct rte_ether_addr *mac_addr,
1677                 uint32_t index,
1678                 uint32_t pool)
1679 {
1680         struct fm10k_macvlan_filter_info *macvlan;
1681
1682         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1683         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1684         macvlan->mac_vmdq_id[index] = pool;
1685         return 0;
1686 }
1687
1688 /* Remove a MAC address, and update filters */
1689 static void
1690 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1691 {
1692         struct rte_eth_dev_data *data = dev->data;
1693         struct fm10k_macvlan_filter_info *macvlan;
1694
1695         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1696         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1697                         FALSE, macvlan->mac_vmdq_id[index]);
1698         macvlan->mac_vmdq_id[index] = 0;
1699 }
1700
1701 static inline int
1702 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1703 {
1704         if ((request < min) || (request > max) || ((request % mult) != 0))
1705                 return -1;
1706         else
1707                 return 0;
1708 }
1709
1710
1711 static inline int
1712 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1713 {
1714         if ((request < min) || (request > max) || ((div % request) != 0))
1715                 return -1;
1716         else
1717                 return 0;
1718 }
1719
1720 static inline int
1721 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1722 {
1723         uint16_t rx_free_thresh;
1724
1725         if (conf->rx_free_thresh == 0)
1726                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1727         else
1728                 rx_free_thresh = conf->rx_free_thresh;
1729
1730         /* make sure the requested threshold satisfies the constraints */
1731         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1732                         FM10K_RX_FREE_THRESH_MAX(q),
1733                         FM10K_RX_FREE_THRESH_DIV(q),
1734                         rx_free_thresh)) {
1735                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1736                         "less than or equal to %u, "
1737                         "greater than or equal to %u, "
1738                         "and a divisor of %u",
1739                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1740                         FM10K_RX_FREE_THRESH_MIN(q),
1741                         FM10K_RX_FREE_THRESH_DIV(q));
1742                 return -EINVAL;
1743         }
1744
1745         q->alloc_thresh = rx_free_thresh;
1746         q->drop_en = conf->rx_drop_en;
1747         q->rx_deferred_start = conf->rx_deferred_start;
1748
1749         return 0;
1750 }
1751
1752 /*
1753  * Hardware requires specific alignment for Rx packet buffers. At
1754  * least one of the following two conditions must be satisfied.
1755  *  1. Address is 512B aligned
1756  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1757  *
1758  * As such, the driver may need to adjust the DMA address within the
1759  * buffer by up to 512B.
1760  *
1761  * return 1 if the element size is valid, otherwise return 0.
1762  */
1763 static int
1764 mempool_element_size_valid(struct rte_mempool *mp)
1765 {
1766         uint32_t min_size;
1767
1768         /* elt_size includes mbuf header and headroom */
1769         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1770                         RTE_PKTMBUF_HEADROOM;
1771
1772         /* account for up to 512B of alignment */
1773         min_size -= FM10K_RX_DATABUF_ALIGN;
1774
1775         /* sanity check for overflow */
1776         if (min_size > mp->elt_size)
1777                 return 0;
1778
1779         /* size is valid */
1780         return 1;
1781 }
1782
1783 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1784 {
1785         RTE_SET_USED(dev);
1786
1787         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1788 }
1789
1790 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1791 {
1792         RTE_SET_USED(dev);
1793
1794         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1795                            DEV_RX_OFFLOAD_VLAN_FILTER |
1796                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1797                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1798                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1799                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1800                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1801 }
1802
1803 static int
1804 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1805         uint16_t nb_desc, unsigned int socket_id,
1806         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1807 {
1808         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1809         struct fm10k_dev_info *dev_info =
1810                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1811         struct fm10k_rx_queue *q;
1812         const struct rte_memzone *mz;
1813         uint64_t offloads;
1814
1815         PMD_INIT_FUNC_TRACE();
1816
1817         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1818
1819         /* make sure the mempool element size can account for alignment. */
1820         if (!mempool_element_size_valid(mp)) {
1821                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1822                 return -EINVAL;
1823         }
1824
1825         /* make sure a valid number of descriptors have been requested */
1826         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1827                                 FM10K_MULT_RX_DESC, nb_desc)) {
1828                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1829                         "less than or equal to %"PRIu32", "
1830                         "greater than or equal to %u, "
1831                         "and a multiple of %u",
1832                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1833                         FM10K_MULT_RX_DESC);
1834                 return -EINVAL;
1835         }
1836
1837         /*
1838          * if this queue existed already, free the associated memory. The
1839          * queue cannot be reused in case we need to allocate memory on
1840          * different socket than was previously used.
1841          */
1842         if (dev->data->rx_queues[queue_id] != NULL) {
1843                 rx_queue_free(dev->data->rx_queues[queue_id]);
1844                 dev->data->rx_queues[queue_id] = NULL;
1845         }
1846
1847         /* allocate memory for the queue structure */
1848         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1849                                 socket_id);
1850         if (q == NULL) {
1851                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1852                 return -ENOMEM;
1853         }
1854
1855         /* setup queue */
1856         q->mp = mp;
1857         q->nb_desc = nb_desc;
1858         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1859         q->port_id = dev->data->port_id;
1860         q->queue_id = queue_id;
1861         q->tail_ptr = (volatile uint32_t *)
1862                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1863         q->offloads = offloads;
1864         if (handle_rxconf(q, conf))
1865                 return -EINVAL;
1866
1867         /* allocate memory for the software ring */
1868         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1869                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1870                         RTE_CACHE_LINE_SIZE, socket_id);
1871         if (q->sw_ring == NULL) {
1872                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1873                 rte_free(q);
1874                 return -ENOMEM;
1875         }
1876
1877         /*
1878          * allocate memory for the hardware descriptor ring. A memzone large
1879          * enough to hold the maximum ring size is requested to allow for
1880          * resizing in later calls to the queue setup function.
1881          */
1882         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1883                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1884                                       socket_id);
1885         if (mz == NULL) {
1886                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1887                 rte_free(q->sw_ring);
1888                 rte_free(q);
1889                 return -ENOMEM;
1890         }
1891         q->hw_ring = mz->addr;
1892         q->hw_ring_phys_addr = mz->iova;
1893
1894         /* Check if number of descs satisfied Vector requirement */
1895         if (!rte_is_power_of_2(nb_desc)) {
1896                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1897                                     "preconditions - canceling the feature for "
1898                                     "the whole port[%d]",
1899                              q->queue_id, q->port_id);
1900                 dev_info->rx_vec_allowed = false;
1901         } else
1902                 fm10k_rxq_vec_setup(q);
1903
1904         dev->data->rx_queues[queue_id] = q;
1905         return 0;
1906 }
1907
1908 static void
1909 fm10k_rx_queue_release(void *queue)
1910 {
1911         PMD_INIT_FUNC_TRACE();
1912
1913         rx_queue_free(queue);
1914 }
1915
1916 static inline int
1917 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1918 {
1919         uint16_t tx_free_thresh;
1920         uint16_t tx_rs_thresh;
1921
1922         /* constraint MACROs require that tx_free_thresh is configured
1923          * before tx_rs_thresh */
1924         if (conf->tx_free_thresh == 0)
1925                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1926         else
1927                 tx_free_thresh = conf->tx_free_thresh;
1928
1929         /* make sure the requested threshold satisfies the constraints */
1930         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1931                         FM10K_TX_FREE_THRESH_MAX(q),
1932                         FM10K_TX_FREE_THRESH_DIV(q),
1933                         tx_free_thresh)) {
1934                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1935                         "less than or equal to %u, "
1936                         "greater than or equal to %u, "
1937                         "and a divisor of %u",
1938                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1939                         FM10K_TX_FREE_THRESH_MIN(q),
1940                         FM10K_TX_FREE_THRESH_DIV(q));
1941                 return -EINVAL;
1942         }
1943
1944         q->free_thresh = tx_free_thresh;
1945
1946         if (conf->tx_rs_thresh == 0)
1947                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1948         else
1949                 tx_rs_thresh = conf->tx_rs_thresh;
1950
1951         q->tx_deferred_start = conf->tx_deferred_start;
1952
1953         /* make sure the requested threshold satisfies the constraints */
1954         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1955                         FM10K_TX_RS_THRESH_MAX(q),
1956                         FM10K_TX_RS_THRESH_DIV(q),
1957                         tx_rs_thresh)) {
1958                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1959                         "less than or equal to %u, "
1960                         "greater than or equal to %u, "
1961                         "and a divisor of %u",
1962                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1963                         FM10K_TX_RS_THRESH_MIN(q),
1964                         FM10K_TX_RS_THRESH_DIV(q));
1965                 return -EINVAL;
1966         }
1967
1968         q->rs_thresh = tx_rs_thresh;
1969
1970         return 0;
1971 }
1972
1973 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1974 {
1975         RTE_SET_USED(dev);
1976
1977         return 0;
1978 }
1979
1980 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1981 {
1982         RTE_SET_USED(dev);
1983
1984         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1985                           DEV_TX_OFFLOAD_MULTI_SEGS  |
1986                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1987                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1988                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1989                           DEV_TX_OFFLOAD_TCP_TSO);
1990 }
1991
1992 static int
1993 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1994         uint16_t nb_desc, unsigned int socket_id,
1995         const struct rte_eth_txconf *conf)
1996 {
1997         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1998         struct fm10k_tx_queue *q;
1999         const struct rte_memzone *mz;
2000         uint64_t offloads;
2001
2002         PMD_INIT_FUNC_TRACE();
2003
2004         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2005
2006         /* make sure a valid number of descriptors have been requested */
2007         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2008                                 FM10K_MULT_TX_DESC, nb_desc)) {
2009                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2010                         "less than or equal to %"PRIu32", "
2011                         "greater than or equal to %u, "
2012                         "and a multiple of %u",
2013                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2014                         FM10K_MULT_TX_DESC);
2015                 return -EINVAL;
2016         }
2017
2018         /*
2019          * if this queue existed already, free the associated memory. The
2020          * queue cannot be reused in case we need to allocate memory on
2021          * different socket than was previously used.
2022          */
2023         if (dev->data->tx_queues[queue_id] != NULL) {
2024                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2025
2026                 tx_queue_free(txq);
2027                 dev->data->tx_queues[queue_id] = NULL;
2028         }
2029
2030         /* allocate memory for the queue structure */
2031         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2032                                 socket_id);
2033         if (q == NULL) {
2034                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2035                 return -ENOMEM;
2036         }
2037
2038         /* setup queue */
2039         q->nb_desc = nb_desc;
2040         q->port_id = dev->data->port_id;
2041         q->queue_id = queue_id;
2042         q->offloads = offloads;
2043         q->ops = &def_txq_ops;
2044         q->tail_ptr = (volatile uint32_t *)
2045                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2046         if (handle_txconf(q, conf))
2047                 return -EINVAL;
2048
2049         /* allocate memory for the software ring */
2050         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2051                                         nb_desc * sizeof(struct rte_mbuf *),
2052                                         RTE_CACHE_LINE_SIZE, socket_id);
2053         if (q->sw_ring == NULL) {
2054                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2055                 rte_free(q);
2056                 return -ENOMEM;
2057         }
2058
2059         /*
2060          * allocate memory for the hardware descriptor ring. A memzone large
2061          * enough to hold the maximum ring size is requested to allow for
2062          * resizing in later calls to the queue setup function.
2063          */
2064         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2065                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2066                                       socket_id);
2067         if (mz == NULL) {
2068                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2069                 rte_free(q->sw_ring);
2070                 rte_free(q);
2071                 return -ENOMEM;
2072         }
2073         q->hw_ring = mz->addr;
2074         q->hw_ring_phys_addr = mz->iova;
2075
2076         /*
2077          * allocate memory for the RS bit tracker. Enough slots to hold the
2078          * descriptor index for each RS bit needing to be set are required.
2079          */
2080         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2081                                 ((nb_desc + 1) / q->rs_thresh) *
2082                                 sizeof(uint16_t),
2083                                 RTE_CACHE_LINE_SIZE, socket_id);
2084         if (q->rs_tracker.list == NULL) {
2085                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2086                 rte_free(q->sw_ring);
2087                 rte_free(q);
2088                 return -ENOMEM;
2089         }
2090
2091         dev->data->tx_queues[queue_id] = q;
2092         return 0;
2093 }
2094
2095 static void
2096 fm10k_tx_queue_release(void *queue)
2097 {
2098         struct fm10k_tx_queue *q = queue;
2099         PMD_INIT_FUNC_TRACE();
2100
2101         tx_queue_free(q);
2102 }
2103
2104 static int
2105 fm10k_reta_update(struct rte_eth_dev *dev,
2106                         struct rte_eth_rss_reta_entry64 *reta_conf,
2107                         uint16_t reta_size)
2108 {
2109         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2110         uint16_t i, j, idx, shift;
2111         uint8_t mask;
2112         uint32_t reta;
2113
2114         PMD_INIT_FUNC_TRACE();
2115
2116         if (reta_size > FM10K_MAX_RSS_INDICES) {
2117                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2118                         "(%d) doesn't match the number hardware can supported "
2119                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2120                 return -EINVAL;
2121         }
2122
2123         /*
2124          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2125          * 128-entries in 32 registers
2126          */
2127         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2128                 idx = i / RTE_RETA_GROUP_SIZE;
2129                 shift = i % RTE_RETA_GROUP_SIZE;
2130                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2131                                 BIT_MASK_PER_UINT32);
2132                 if (mask == 0)
2133                         continue;
2134
2135                 reta = 0;
2136                 if (mask != BIT_MASK_PER_UINT32)
2137                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2138
2139                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2140                         if (mask & (0x1 << j)) {
2141                                 if (mask != 0xF)
2142                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2143                                 reta |= reta_conf[idx].reta[shift + j] <<
2144                                                 (CHAR_BIT * j);
2145                         }
2146                 }
2147                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2148         }
2149
2150         return 0;
2151 }
2152
2153 static int
2154 fm10k_reta_query(struct rte_eth_dev *dev,
2155                         struct rte_eth_rss_reta_entry64 *reta_conf,
2156                         uint16_t reta_size)
2157 {
2158         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2159         uint16_t i, j, idx, shift;
2160         uint8_t mask;
2161         uint32_t reta;
2162
2163         PMD_INIT_FUNC_TRACE();
2164
2165         if (reta_size < FM10K_MAX_RSS_INDICES) {
2166                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2167                         "(%d) doesn't match the number hardware can supported "
2168                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2169                 return -EINVAL;
2170         }
2171
2172         /*
2173          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2174          * 128-entries in 32 registers
2175          */
2176         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2177                 idx = i / RTE_RETA_GROUP_SIZE;
2178                 shift = i % RTE_RETA_GROUP_SIZE;
2179                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2180                                 BIT_MASK_PER_UINT32);
2181                 if (mask == 0)
2182                         continue;
2183
2184                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2185                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2186                         if (mask & (0x1 << j))
2187                                 reta_conf[idx].reta[shift + j] = ((reta >>
2188                                         CHAR_BIT * j) & UINT8_MAX);
2189                 }
2190         }
2191
2192         return 0;
2193 }
2194
2195 static int
2196 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2197         struct rte_eth_rss_conf *rss_conf)
2198 {
2199         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2200         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2201         uint32_t mrqc;
2202         uint64_t hf = rss_conf->rss_hf;
2203         int i;
2204
2205         PMD_INIT_FUNC_TRACE();
2206
2207         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2208                                 FM10K_RSSRK_ENTRIES_PER_REG))
2209                 return -EINVAL;
2210
2211         if (hf == 0)
2212                 return -EINVAL;
2213
2214         mrqc = 0;
2215         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2216         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2217         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2218         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2219         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2220         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2221         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2222         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2223         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2224
2225         /* If the mapping doesn't fit any supported, return */
2226         if (mrqc == 0)
2227                 return -EINVAL;
2228
2229         if (key != NULL)
2230                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2231                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2232
2233         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2234
2235         return 0;
2236 }
2237
2238 static int
2239 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2240         struct rte_eth_rss_conf *rss_conf)
2241 {
2242         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2243         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2244         uint32_t mrqc;
2245         uint64_t hf;
2246         int i;
2247
2248         PMD_INIT_FUNC_TRACE();
2249
2250         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2251                                 FM10K_RSSRK_ENTRIES_PER_REG))
2252                 return -EINVAL;
2253
2254         if (key != NULL)
2255                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2256                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2257
2258         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2259         hf = 0;
2260         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2261         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2262         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2263         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2264         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2265         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2266         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2267         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2268         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2269
2270         rss_conf->rss_hf = hf;
2271
2272         return 0;
2273 }
2274
2275 static void
2276 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2277 {
2278         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2279         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2280
2281         /* Bind all local non-queue interrupt to vector 0 */
2282         int_map |= FM10K_MISC_VEC_ID;
2283
2284         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2285         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2286         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2287         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2288         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2289         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2290
2291         /* Enable misc causes */
2292         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2293                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2294                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2295                                 FM10K_EIMR_ENABLE(MAILBOX) |
2296                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2297                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2298                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2299                                 FM10K_EIMR_ENABLE(VFLR));
2300
2301         /* Enable ITR 0 */
2302         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2303                                         FM10K_ITR_MASK_CLEAR);
2304         FM10K_WRITE_FLUSH(hw);
2305 }
2306
2307 static void
2308 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2309 {
2310         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2311         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2312
2313         int_map |= FM10K_MISC_VEC_ID;
2314
2315         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2316         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2317         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2318         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2319         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2320         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2321
2322         /* Disable misc causes */
2323         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2324                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2325                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2326                                 FM10K_EIMR_DISABLE(MAILBOX) |
2327                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2328                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2329                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2330                                 FM10K_EIMR_DISABLE(VFLR));
2331
2332         /* Disable ITR 0 */
2333         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2334         FM10K_WRITE_FLUSH(hw);
2335 }
2336
2337 static void
2338 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2339 {
2340         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2341         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2342
2343         /* Bind all local non-queue interrupt to vector 0 */
2344         int_map |= FM10K_MISC_VEC_ID;
2345
2346         /* Only INT 0 available, other 15 are reserved. */
2347         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2348
2349         /* Enable ITR 0 */
2350         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2351                                         FM10K_ITR_MASK_CLEAR);
2352         FM10K_WRITE_FLUSH(hw);
2353 }
2354
2355 static void
2356 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2357 {
2358         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2359         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2360
2361         int_map |= FM10K_MISC_VEC_ID;
2362
2363         /* Only INT 0 available, other 15 are reserved. */
2364         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2365
2366         /* Disable ITR 0 */
2367         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2368         FM10K_WRITE_FLUSH(hw);
2369 }
2370
2371 static int
2372 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2373 {
2374         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2375         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2376
2377         /* Enable ITR */
2378         if (hw->mac.type == fm10k_mac_pf)
2379                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2380                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2381         else
2382                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2383                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2384         rte_intr_enable(&pdev->intr_handle);
2385         return 0;
2386 }
2387
2388 static int
2389 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2390 {
2391         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2392         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2393
2394         /* Disable ITR */
2395         if (hw->mac.type == fm10k_mac_pf)
2396                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2397                         FM10K_ITR_MASK_SET);
2398         else
2399                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2400                         FM10K_ITR_MASK_SET);
2401         return 0;
2402 }
2403
2404 static int
2405 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2406 {
2407         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2408         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2409         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2410         uint32_t intr_vector, vec;
2411         uint16_t queue_id;
2412         int result = 0;
2413
2414         /* fm10k needs one separate interrupt for mailbox,
2415          * so only drivers which support multiple interrupt vectors
2416          * e.g. vfio-pci can work for fm10k interrupt mode
2417          */
2418         if (!rte_intr_cap_multiple(intr_handle) ||
2419                         dev->data->dev_conf.intr_conf.rxq == 0)
2420                 return result;
2421
2422         intr_vector = dev->data->nb_rx_queues;
2423
2424         /* disable interrupt first */
2425         rte_intr_disable(intr_handle);
2426         if (hw->mac.type == fm10k_mac_pf)
2427                 fm10k_dev_disable_intr_pf(dev);
2428         else
2429                 fm10k_dev_disable_intr_vf(dev);
2430
2431         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2432                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2433                 result = -EIO;
2434         }
2435
2436         if (rte_intr_dp_is_en(intr_handle) && !result) {
2437                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2438                         dev->data->nb_rx_queues * sizeof(int), 0);
2439                 if (intr_handle->intr_vec) {
2440                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2441                                         queue_id < dev->data->nb_rx_queues;
2442                                         queue_id++) {
2443                                 intr_handle->intr_vec[queue_id] = vec;
2444                                 if (vec < intr_handle->nb_efd - 1
2445                                                 + FM10K_RX_VEC_START)
2446                                         vec++;
2447                         }
2448                 } else {
2449                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2450                                 " intr_vec", dev->data->nb_rx_queues);
2451                         rte_intr_efd_disable(intr_handle);
2452                         result = -ENOMEM;
2453                 }
2454         }
2455
2456         if (hw->mac.type == fm10k_mac_pf)
2457                 fm10k_dev_enable_intr_pf(dev);
2458         else
2459                 fm10k_dev_enable_intr_vf(dev);
2460         rte_intr_enable(intr_handle);
2461         hw->mac.ops.update_int_moderator(hw);
2462         return result;
2463 }
2464
2465 static int
2466 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2467 {
2468         struct fm10k_fault fault;
2469         int err;
2470         const char *estr = "Unknown error";
2471
2472         /* Process PCA fault */
2473         if (eicr & FM10K_EICR_PCA_FAULT) {
2474                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2475                 if (err)
2476                         goto error;
2477                 switch (fault.type) {
2478                 case PCA_NO_FAULT:
2479                         estr = "PCA_NO_FAULT"; break;
2480                 case PCA_UNMAPPED_ADDR:
2481                         estr = "PCA_UNMAPPED_ADDR"; break;
2482                 case PCA_BAD_QACCESS_PF:
2483                         estr = "PCA_BAD_QACCESS_PF"; break;
2484                 case PCA_BAD_QACCESS_VF:
2485                         estr = "PCA_BAD_QACCESS_VF"; break;
2486                 case PCA_MALICIOUS_REQ:
2487                         estr = "PCA_MALICIOUS_REQ"; break;
2488                 case PCA_POISONED_TLP:
2489                         estr = "PCA_POISONED_TLP"; break;
2490                 case PCA_TLP_ABORT:
2491                         estr = "PCA_TLP_ABORT"; break;
2492                 default:
2493                         goto error;
2494                 }
2495                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2496                         estr, fault.func ? "VF" : "PF", fault.func,
2497                         fault.address, fault.specinfo);
2498         }
2499
2500         /* Process THI fault */
2501         if (eicr & FM10K_EICR_THI_FAULT) {
2502                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2503                 if (err)
2504                         goto error;
2505                 switch (fault.type) {
2506                 case THI_NO_FAULT:
2507                         estr = "THI_NO_FAULT"; break;
2508                 case THI_MAL_DIS_Q_FAULT:
2509                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2510                 default:
2511                         goto error;
2512                 }
2513                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2514                         estr, fault.func ? "VF" : "PF", fault.func,
2515                         fault.address, fault.specinfo);
2516         }
2517
2518         /* Process FUM fault */
2519         if (eicr & FM10K_EICR_FUM_FAULT) {
2520                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2521                 if (err)
2522                         goto error;
2523                 switch (fault.type) {
2524                 case FUM_NO_FAULT:
2525                         estr = "FUM_NO_FAULT"; break;
2526                 case FUM_UNMAPPED_ADDR:
2527                         estr = "FUM_UNMAPPED_ADDR"; break;
2528                 case FUM_POISONED_TLP:
2529                         estr = "FUM_POISONED_TLP"; break;
2530                 case FUM_BAD_VF_QACCESS:
2531                         estr = "FUM_BAD_VF_QACCESS"; break;
2532                 case FUM_ADD_DECODE_ERR:
2533                         estr = "FUM_ADD_DECODE_ERR"; break;
2534                 case FUM_RO_ERROR:
2535                         estr = "FUM_RO_ERROR"; break;
2536                 case FUM_QPRC_CRC_ERROR:
2537                         estr = "FUM_QPRC_CRC_ERROR"; break;
2538                 case FUM_CSR_TIMEOUT:
2539                         estr = "FUM_CSR_TIMEOUT"; break;
2540                 case FUM_INVALID_TYPE:
2541                         estr = "FUM_INVALID_TYPE"; break;
2542                 case FUM_INVALID_LENGTH:
2543                         estr = "FUM_INVALID_LENGTH"; break;
2544                 case FUM_INVALID_BE:
2545                         estr = "FUM_INVALID_BE"; break;
2546                 case FUM_INVALID_ALIGN:
2547                         estr = "FUM_INVALID_ALIGN"; break;
2548                 default:
2549                         goto error;
2550                 }
2551                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2552                         estr, fault.func ? "VF" : "PF", fault.func,
2553                         fault.address, fault.specinfo);
2554         }
2555
2556         return 0;
2557 error:
2558         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2559         return err;
2560 }
2561
2562 /**
2563  * PF interrupt handler triggered by NIC for handling specific interrupt.
2564  *
2565  * @param handle
2566  *  Pointer to interrupt handle.
2567  * @param param
2568  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2569  *
2570  * @return
2571  *  void
2572  */
2573 static void
2574 fm10k_dev_interrupt_handler_pf(void *param)
2575 {
2576         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2577         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2578         uint32_t cause, status;
2579         struct fm10k_dev_info *dev_info =
2580                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2581         int status_mbx;
2582         s32 err;
2583
2584         if (hw->mac.type != fm10k_mac_pf)
2585                 return;
2586
2587         cause = FM10K_READ_REG(hw, FM10K_EICR);
2588
2589         /* Handle PCI fault cases */
2590         if (cause & FM10K_EICR_FAULT_MASK) {
2591                 PMD_INIT_LOG(ERR, "INT: find fault!");
2592                 fm10k_dev_handle_fault(hw, cause);
2593         }
2594
2595         /* Handle switch up/down */
2596         if (cause & FM10K_EICR_SWITCHNOTREADY)
2597                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2598
2599         if (cause & FM10K_EICR_SWITCHREADY) {
2600                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2601                 if (dev_info->sm_down == 1) {
2602                         fm10k_mbx_lock(hw);
2603
2604                         /* For recreating logical ports */
2605                         status_mbx = hw->mac.ops.update_lport_state(hw,
2606                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2607                         if (status_mbx == FM10K_SUCCESS)
2608                                 PMD_INIT_LOG(INFO,
2609                                         "INT: Recreated Logical port");
2610                         else
2611                                 PMD_INIT_LOG(INFO,
2612                                         "INT: Logical ports weren't recreated");
2613
2614                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2615                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2616                         if (status_mbx != FM10K_SUCCESS)
2617                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2618
2619                         fm10k_mbx_unlock(hw);
2620
2621                         /* first clear the internal SW recording structure */
2622                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2623                                                 ETH_MQ_RX_VMDQ_FLAG))
2624                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2625                                         false);
2626
2627                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2628                                         MAIN_VSI_POOL_NUMBER);
2629
2630                         /*
2631                          * Add default mac address and vlan for the logical
2632                          * ports that have been created, leave to the
2633                          * application to fully recover Rx filtering.
2634                          */
2635                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2636                                         MAIN_VSI_POOL_NUMBER);
2637
2638                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2639                                                 ETH_MQ_RX_VMDQ_FLAG))
2640                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2641                                         true);
2642
2643                         dev_info->sm_down = 0;
2644                         _rte_eth_dev_callback_process(dev,
2645                                         RTE_ETH_EVENT_INTR_LSC,
2646                                         NULL);
2647                 }
2648         }
2649
2650         /* Handle mailbox message */
2651         fm10k_mbx_lock(hw);
2652         err = hw->mbx.ops.process(hw, &hw->mbx);
2653         fm10k_mbx_unlock(hw);
2654
2655         if (err == FM10K_ERR_RESET_REQUESTED) {
2656                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2657                 dev_info->sm_down = 1;
2658                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2659                                 NULL);
2660         }
2661
2662         /* Handle SRAM error */
2663         if (cause & FM10K_EICR_SRAMERROR) {
2664                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2665
2666                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2667                 /* Write to clear pending bits */
2668                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2669
2670                 /* Todo: print out error message after shared code  updates */
2671         }
2672
2673         /* Clear these 3 events if having any */
2674         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2675                  FM10K_EICR_SWITCHREADY;
2676         if (cause)
2677                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2678
2679         /* Re-enable interrupt from device side */
2680         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2681                                         FM10K_ITR_MASK_CLEAR);
2682         /* Re-enable interrupt from host side */
2683         rte_intr_enable(dev->intr_handle);
2684 }
2685
2686 /**
2687  * VF interrupt handler triggered by NIC for handling specific interrupt.
2688  *
2689  * @param handle
2690  *  Pointer to interrupt handle.
2691  * @param param
2692  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2693  *
2694  * @return
2695  *  void
2696  */
2697 static void
2698 fm10k_dev_interrupt_handler_vf(void *param)
2699 {
2700         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2701         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2702         struct fm10k_mbx_info *mbx = &hw->mbx;
2703         struct fm10k_dev_info *dev_info =
2704                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2705         const enum fm10k_mbx_state state = mbx->state;
2706         int status_mbx;
2707
2708         if (hw->mac.type != fm10k_mac_vf)
2709                 return;
2710
2711         /* Handle mailbox message if lock is acquired */
2712         fm10k_mbx_lock(hw);
2713         hw->mbx.ops.process(hw, &hw->mbx);
2714         fm10k_mbx_unlock(hw);
2715
2716         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2717                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2718
2719                 fm10k_mbx_lock(hw);
2720                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2721                                 MAX_LPORT_NUM, 1);
2722                 fm10k_mbx_unlock(hw);
2723
2724                 /* Setting reset flag */
2725                 dev_info->sm_down = 1;
2726                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2727                                 NULL);
2728         }
2729
2730         if (dev_info->sm_down == 1 &&
2731                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2732                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2733                 fm10k_mbx_lock(hw);
2734                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2735                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2736                 if (status_mbx != FM10K_SUCCESS)
2737                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2738                 fm10k_mbx_unlock(hw);
2739
2740                 /* first clear the internal SW recording structure */
2741                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2742                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2743                                 MAIN_VSI_POOL_NUMBER);
2744
2745                 /*
2746                  * Add default mac address and vlan for the logical ports that
2747                  * have been created, leave to the application to fully recover
2748                  * Rx filtering.
2749                  */
2750                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2751                                 MAIN_VSI_POOL_NUMBER);
2752                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2753
2754                 dev_info->sm_down = 0;
2755                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2756                                 NULL);
2757         }
2758
2759         /* Re-enable interrupt from device side */
2760         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2761                                         FM10K_ITR_MASK_CLEAR);
2762         /* Re-enable interrupt from host side */
2763         rte_intr_enable(dev->intr_handle);
2764 }
2765
2766 /* Mailbox message handler in VF */
2767 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2768         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2769         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2770         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2771         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2772 };
2773
2774 static int
2775 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2776 {
2777         int err = 0;
2778
2779         /* Initialize mailbox lock */
2780         fm10k_mbx_initlock(hw);
2781
2782         /* Replace default message handler with new ones */
2783         if (hw->mac.type == fm10k_mac_vf)
2784                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2785
2786         if (err) {
2787                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2788                                 err);
2789                 return err;
2790         }
2791         /* Connect to SM for PF device or PF for VF device */
2792         return hw->mbx.ops.connect(hw, &hw->mbx);
2793 }
2794
2795 static void
2796 fm10k_close_mbx_service(struct fm10k_hw *hw)
2797 {
2798         /* Disconnect from SM for PF device or PF for VF device */
2799         hw->mbx.ops.disconnect(hw, &hw->mbx);
2800 }
2801
2802 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2803         .dev_configure          = fm10k_dev_configure,
2804         .dev_start              = fm10k_dev_start,
2805         .dev_stop               = fm10k_dev_stop,
2806         .dev_close              = fm10k_dev_close,
2807         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2808         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2809         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2810         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2811         .stats_get              = fm10k_stats_get,
2812         .xstats_get             = fm10k_xstats_get,
2813         .xstats_get_names       = fm10k_xstats_get_names,
2814         .stats_reset            = fm10k_stats_reset,
2815         .xstats_reset           = fm10k_stats_reset,
2816         .link_update            = fm10k_link_update,
2817         .dev_infos_get          = fm10k_dev_infos_get,
2818         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2819         .vlan_filter_set        = fm10k_vlan_filter_set,
2820         .vlan_offload_set       = fm10k_vlan_offload_set,
2821         .mac_addr_add           = fm10k_macaddr_add,
2822         .mac_addr_remove        = fm10k_macaddr_remove,
2823         .rx_queue_start         = fm10k_dev_rx_queue_start,
2824         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2825         .tx_queue_start         = fm10k_dev_tx_queue_start,
2826         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2827         .rx_queue_setup         = fm10k_rx_queue_setup,
2828         .rx_queue_release       = fm10k_rx_queue_release,
2829         .tx_queue_setup         = fm10k_tx_queue_setup,
2830         .tx_queue_release       = fm10k_tx_queue_release,
2831         .rx_queue_count         = fm10k_dev_rx_queue_count,
2832         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2833         .rx_descriptor_status = fm10k_dev_rx_descriptor_status,
2834         .tx_descriptor_status = fm10k_dev_tx_descriptor_status,
2835         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2836         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2837         .reta_update            = fm10k_reta_update,
2838         .reta_query             = fm10k_reta_query,
2839         .rss_hash_update        = fm10k_rss_hash_update,
2840         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2841 };
2842
2843 static int ftag_check_handler(__rte_unused const char *key,
2844                 const char *value, __rte_unused void *opaque)
2845 {
2846         if (strcmp(value, "1"))
2847                 return -1;
2848
2849         return 0;
2850 }
2851
2852 static int
2853 fm10k_check_ftag(struct rte_devargs *devargs)
2854 {
2855         struct rte_kvargs *kvlist;
2856         const char *ftag_key = "enable_ftag";
2857
2858         if (devargs == NULL)
2859                 return 0;
2860
2861         kvlist = rte_kvargs_parse(devargs->args, NULL);
2862         if (kvlist == NULL)
2863                 return 0;
2864
2865         if (!rte_kvargs_count(kvlist, ftag_key)) {
2866                 rte_kvargs_free(kvlist);
2867                 return 0;
2868         }
2869         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2870         if (rte_kvargs_process(kvlist, ftag_key,
2871                                 ftag_check_handler, NULL) < 0) {
2872                 rte_kvargs_free(kvlist);
2873                 return 0;
2874         }
2875         rte_kvargs_free(kvlist);
2876
2877         return 1;
2878 }
2879
2880 static uint16_t
2881 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2882                     uint16_t nb_pkts)
2883 {
2884         uint16_t nb_tx = 0;
2885         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2886
2887         while (nb_pkts) {
2888                 uint16_t ret, num;
2889
2890                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2891                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2892                                                  num);
2893                 nb_tx += ret;
2894                 nb_pkts -= ret;
2895                 if (ret < num)
2896                         break;
2897         }
2898
2899         return nb_tx;
2900 }
2901
2902 static void __attribute__((cold))
2903 fm10k_set_tx_function(struct rte_eth_dev *dev)
2904 {
2905         struct fm10k_tx_queue *txq;
2906         int i;
2907         int use_sse = 1;
2908         uint16_t tx_ftag_en = 0;
2909
2910         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2911                 /* primary process has set the ftag flag and offloads */
2912                 txq = dev->data->tx_queues[0];
2913                 if (fm10k_tx_vec_condition_check(txq)) {
2914                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2915                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2916                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2917                 } else {
2918                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2919                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2920                         dev->tx_pkt_prepare = NULL;
2921                 }
2922                 return;
2923         }
2924
2925         if (fm10k_check_ftag(dev->device->devargs))
2926                 tx_ftag_en = 1;
2927
2928         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2929                 txq = dev->data->tx_queues[i];
2930                 txq->tx_ftag_en = tx_ftag_en;
2931                 /* Check if Vector Tx is satisfied */
2932                 if (fm10k_tx_vec_condition_check(txq))
2933                         use_sse = 0;
2934         }
2935
2936         if (use_sse) {
2937                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2938                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2939                         txq = dev->data->tx_queues[i];
2940                         fm10k_txq_vec_setup(txq);
2941                 }
2942                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2943                 dev->tx_pkt_prepare = NULL;
2944         } else {
2945                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2946                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2947                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2948         }
2949 }
2950
2951 static void __attribute__((cold))
2952 fm10k_set_rx_function(struct rte_eth_dev *dev)
2953 {
2954         struct fm10k_dev_info *dev_info =
2955                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2956         uint16_t i, rx_using_sse;
2957         uint16_t rx_ftag_en = 0;
2958
2959         if (fm10k_check_ftag(dev->device->devargs))
2960                 rx_ftag_en = 1;
2961
2962         /* In order to allow Vector Rx there are a few configuration
2963          * conditions to be met.
2964          */
2965         if (!fm10k_rx_vec_condition_check(dev) &&
2966                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2967                 if (dev->data->scattered_rx)
2968                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2969                 else
2970                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2971         } else if (dev->data->scattered_rx)
2972                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2973         else
2974                 dev->rx_pkt_burst = fm10k_recv_pkts;
2975
2976         rx_using_sse =
2977                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2978                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2979
2980         if (rx_using_sse)
2981                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2982         else
2983                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2984
2985         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2986                 return;
2987
2988         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2989                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2990
2991                 rxq->rx_using_sse = rx_using_sse;
2992                 rxq->rx_ftag_en = rx_ftag_en;
2993         }
2994 }
2995
2996 static void
2997 fm10k_params_init(struct rte_eth_dev *dev)
2998 {
2999         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3000         struct fm10k_dev_info *info =
3001                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3002
3003         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3004          * there is no way to get link status without reading BAR4.  Until this
3005          * works, assume we have maximum bandwidth.
3006          * @todo - fix bus info
3007          */
3008         hw->bus_caps.speed = fm10k_bus_speed_8000;
3009         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3010         hw->bus_caps.payload = fm10k_bus_payload_512;
3011         hw->bus.speed = fm10k_bus_speed_8000;
3012         hw->bus.width = fm10k_bus_width_pcie_x8;
3013         hw->bus.payload = fm10k_bus_payload_256;
3014
3015         info->rx_vec_allowed = true;
3016         info->sm_down = false;
3017 }
3018
3019 static int
3020 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3021 {
3022         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3023         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3024         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3025         int diag, i;
3026         struct fm10k_macvlan_filter_info *macvlan;
3027
3028         PMD_INIT_FUNC_TRACE();
3029
3030         dev->dev_ops = &fm10k_eth_dev_ops;
3031         dev->rx_pkt_burst = &fm10k_recv_pkts;
3032         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3033         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3034
3035         /*
3036          * Primary process does the whole initialization, for secondary
3037          * processes, we just select the same Rx and Tx function as primary.
3038          */
3039         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3040                 fm10k_set_rx_function(dev);
3041                 fm10k_set_tx_function(dev);
3042                 return 0;
3043         }
3044
3045         rte_eth_copy_pci_info(dev, pdev);
3046
3047         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3048         memset(macvlan, 0, sizeof(*macvlan));
3049         /* Vendor and Device ID need to be set before init of shared code */
3050         memset(hw, 0, sizeof(*hw));
3051         hw->device_id = pdev->id.device_id;
3052         hw->vendor_id = pdev->id.vendor_id;
3053         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3054         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3055         hw->revision_id = 0;
3056         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3057         if (hw->hw_addr == NULL) {
3058                 PMD_INIT_LOG(ERR, "Bad mem resource."
3059                         " Try to blacklist unused devices.");
3060                 return -EIO;
3061         }
3062
3063         /* Store fm10k_adapter pointer */
3064         hw->back = dev->data->dev_private;
3065
3066         /* Initialize the shared code */
3067         diag = fm10k_init_shared_code(hw);
3068         if (diag != FM10K_SUCCESS) {
3069                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3070                 return -EIO;
3071         }
3072
3073         /* Initialize parameters */
3074         fm10k_params_init(dev);
3075
3076         /* Initialize the hw */
3077         diag = fm10k_init_hw(hw);
3078         if (diag != FM10K_SUCCESS) {
3079                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3080                 return -EIO;
3081         }
3082
3083         /* Initialize MAC address(es) */
3084         dev->data->mac_addrs = rte_zmalloc("fm10k",
3085                         RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3086         if (dev->data->mac_addrs == NULL) {
3087                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3088                 return -ENOMEM;
3089         }
3090
3091         diag = fm10k_read_mac_addr(hw);
3092
3093         rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3094                         &dev->data->mac_addrs[0]);
3095
3096         if (diag != FM10K_SUCCESS ||
3097                 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3098
3099                 /* Generate a random addr */
3100                 rte_eth_random_addr(hw->mac.addr);
3101                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3102                 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr,
3103                 &dev->data->mac_addrs[0]);
3104         }
3105
3106         /* Reset the hw statistics */
3107         fm10k_stats_reset(dev);
3108
3109         /* Reset the hw */
3110         diag = fm10k_reset_hw(hw);
3111         if (diag != FM10K_SUCCESS) {
3112                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3113                 return -EIO;
3114         }
3115
3116         /* Setup mailbox service */
3117         diag = fm10k_setup_mbx_service(hw);
3118         if (diag != FM10K_SUCCESS) {
3119                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3120                 return -EIO;
3121         }
3122
3123         /*PF/VF has different interrupt handling mechanism */
3124         if (hw->mac.type == fm10k_mac_pf) {
3125                 /* register callback func to eal lib */
3126                 rte_intr_callback_register(intr_handle,
3127                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3128
3129                 /* enable MISC interrupt */
3130                 fm10k_dev_enable_intr_pf(dev);
3131         } else { /* VF */
3132                 rte_intr_callback_register(intr_handle,
3133                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3134
3135                 fm10k_dev_enable_intr_vf(dev);
3136         }
3137
3138         /* Enable intr after callback registered */
3139         rte_intr_enable(intr_handle);
3140
3141         hw->mac.ops.update_int_moderator(hw);
3142
3143         /* Make sure Switch Manager is ready before going forward. */
3144         if (hw->mac.type == fm10k_mac_pf) {
3145                 int switch_ready = 0;
3146
3147                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3148                         fm10k_mbx_lock(hw);
3149                         hw->mac.ops.get_host_state(hw, &switch_ready);
3150                         fm10k_mbx_unlock(hw);
3151                         if (switch_ready)
3152                                 break;
3153                         /* Delay some time to acquire async LPORT_MAP info. */
3154                         rte_delay_us(WAIT_SWITCH_MSG_US);
3155                 }
3156
3157                 if (switch_ready == 0) {
3158                         PMD_INIT_LOG(ERR, "switch is not ready");
3159                         return -1;
3160                 }
3161         }
3162
3163         /*
3164          * Below function will trigger operations on mailbox, acquire lock to
3165          * avoid race condition from interrupt handler. Operations on mailbox
3166          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3167          * will handle and generate an interrupt to our side. Then,  FIFO in
3168          * mailbox will be touched.
3169          */
3170         fm10k_mbx_lock(hw);
3171         /* Enable port first */
3172         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3173                                         MAX_LPORT_NUM, 1);
3174
3175         /* Set unicast mode by default. App can change to other mode in other
3176          * API func.
3177          */
3178         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3179                                         FM10K_XCAST_MODE_NONE);
3180
3181         fm10k_mbx_unlock(hw);
3182
3183         /* Make sure default VID is ready before going forward. */
3184         if (hw->mac.type == fm10k_mac_pf) {
3185                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3186                         if (hw->mac.default_vid)
3187                                 break;
3188                         /* Delay some time to acquire async port VLAN info. */
3189                         rte_delay_us(WAIT_SWITCH_MSG_US);
3190                 }
3191
3192                 if (!hw->mac.default_vid) {
3193                         PMD_INIT_LOG(ERR, "default VID is not ready");
3194                         return -1;
3195                 }
3196         }
3197
3198         /* Add default mac address */
3199         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3200                 MAIN_VSI_POOL_NUMBER);
3201
3202         return 0;
3203 }
3204
3205 static int
3206 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3207 {
3208         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3209         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3210         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3211         PMD_INIT_FUNC_TRACE();
3212
3213         /* only uninitialize in the primary process */
3214         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3215                 return 0;
3216
3217         /* safe to close dev here */
3218         fm10k_dev_close(dev);
3219
3220         dev->dev_ops = NULL;
3221         dev->rx_pkt_burst = NULL;
3222         dev->tx_pkt_burst = NULL;
3223
3224         /* disable uio/vfio intr */
3225         rte_intr_disable(intr_handle);
3226
3227         /*PF/VF has different interrupt handling mechanism */
3228         if (hw->mac.type == fm10k_mac_pf) {
3229                 /* disable interrupt */
3230                 fm10k_dev_disable_intr_pf(dev);
3231
3232                 /* unregister callback func to eal lib */
3233                 rte_intr_callback_unregister(intr_handle,
3234                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3235         } else {
3236                 /* disable interrupt */
3237                 fm10k_dev_disable_intr_vf(dev);
3238
3239                 rte_intr_callback_unregister(intr_handle,
3240                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3241         }
3242
3243         return 0;
3244 }
3245
3246 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3247         struct rte_pci_device *pci_dev)
3248 {
3249         return rte_eth_dev_pci_generic_probe(pci_dev,
3250                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3251 }
3252
3253 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3254 {
3255         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3256 }
3257
3258 /*
3259  * The set of PCI devices this driver supports. This driver will enable both PF
3260  * and SRIOV-VF devices.
3261  */
3262 static const struct rte_pci_id pci_id_fm10k_map[] = {
3263         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3264         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3265         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3266         { .vendor_id = 0, /* sentinel */ },
3267 };
3268
3269 static struct rte_pci_driver rte_pmd_fm10k = {
3270         .id_table = pci_id_fm10k_map,
3271         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3272                      RTE_PCI_DRV_IOVA_AS_VA,
3273         .probe = eth_fm10k_pci_probe,
3274         .remove = eth_fm10k_pci_remove,
3275 };
3276
3277 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3278 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3279 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3280
3281 RTE_INIT(fm10k_init_log)
3282 {
3283         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3284         if (fm10k_logtype_init >= 0)
3285                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3286         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3287         if (fm10k_logtype_driver >= 0)
3288                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3289 }