drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451
 452         PMD_INIT_FUNC_TRACE();
 453
 454         if ((dev->data->dev_conf.rxmode.offloads &
 455              DEV_RX_OFFLOAD_CRC_STRIP) == 0)
 456                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 457
 458         /* multipe queue mode checking */
 459         ret  = fm10k_check_mq_mode(dev);
 460         if (ret != 0) {
 461                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 462                             ret);
 463                 return ret;
 464         }
 465
 466         dev->data->scattered_rx = 0;
 467
 468         return 0;
 469 }
 470
 471 /* fls = find last set bit = 32 minus the number of leading zeros */
 472 #ifndef fls
 473 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 474 #endif
 475
 476 static void
 477 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 478 {
 479         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 480         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 481         uint32_t i;
 482
 483         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 484
 485         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 486                 if (!vmdq_conf->pool_map[i].pools)
 487                         continue;
 488                 fm10k_mbx_lock(hw);
 489                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 490                 fm10k_mbx_unlock(hw);
 491         }
 492 }
 493
 494 static void
 495 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 496 {
 497         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 498
 499         /* Add default mac address */
 500         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 501                 MAIN_VSI_POOL_NUMBER);
 502 }
 503
 504 static void
 505 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 506 {
 507         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 508         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 509         uint32_t mrqc, *key, i, reta, j;
 510         uint64_t hf;
 511
 512 #define RSS_KEY_SIZE 40
 513         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 514                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 515                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 516                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 517                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 518                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 519         };
 520
 521         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 522                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 523                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 524                 return;
 525         }
 526
 527         /* random key is rss_intel_key (default) or user provided (rss_key) */
 528         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 529                 key = (uint32_t *)rss_intel_key;
 530         else
 531                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 532
 533         /* Now fill our hash function seeds, 4 bytes at a time */
 534         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 535                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 536
 537         /*
 538          * Fill in redirection table
 539          * The byte-swap is needed because NIC registers are in
 540          * little-endian order.
 541          */
 542         reta = 0;
 543         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 544                 if (j == dev->data->nb_rx_queues)
 545                         j = 0;
 546                 reta = (reta << CHAR_BIT) | j;
 547                 if ((i & 3) == 3)
 548                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 549                                         rte_bswap32(reta));
 550         }
 551
 552         /*
 553          * Generate RSS hash based on packet types, TCP/UDP
 554          * port numbers and/or IPv4/v6 src and dst addresses
 555          */
 556         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 557         mrqc = 0;
 558         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 559         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 560         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 561         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 562         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 563         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 564         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 565         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 566         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 567
 568         if (mrqc == 0) {
 569                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 570                         "supported", hf);
 571                 return;
 572         }
 573
 574         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 575 }
 576
 577 static void
 578 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 579 {
 580         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 581         uint32_t i;
 582
 583         for (i = 0; i < nb_lport_new; i++) {
 584                 /* Set unicast mode by default. App can change
 585                  * to other mode in other API func.
 586                  */
 587                 fm10k_mbx_lock(hw);
 588                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 589                         FM10K_XCAST_MODE_NONE);
 590                 fm10k_mbx_unlock(hw);
 591         }
 592 }
 593
 594 static void
 595 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 596 {
 597         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 598         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 599         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 600         struct fm10k_macvlan_filter_info *macvlan;
 601         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 602         uint16_t nb_lport_new;
 603
 604         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 605         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 606
 607         fm10k_dev_rss_configure(dev);
 608
 609         /* only PF supports VMDQ */
 610         if (hw->mac.type != fm10k_mac_pf)
 611                 return;
 612
 613         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 614                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 615
 616         /* no pool number change, no need to update logic port and VLAN/MAC */
 617         if (macvlan->nb_queue_pools == nb_queue_pools)
 618                 return;
 619
 620         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 621         fm10k_dev_logic_port_update(dev, nb_lport_new);
 622
 623         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 624         memset(dev->data->mac_addrs, 0,
 625                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 626         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 627                 &dev->data->mac_addrs[0]);
 628         memset(macvlan, 0, sizeof(*macvlan));
 629         macvlan->nb_queue_pools = nb_queue_pools;
 630
 631         if (nb_queue_pools)
 632                 fm10k_dev_vmdq_rx_configure(dev);
 633         else
 634                 fm10k_dev_pf_main_vsi_reset(dev);
 635 }
 636
 637 static int
 638 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 639 {
 640         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 641         int i, ret;
 642         struct fm10k_tx_queue *txq;
 643         uint64_t base_addr;
 644         uint32_t size;
 645
 646         /* Disable TXINT to avoid possible interrupt */
 647         for (i = 0; i < hw->mac.max_queues; i++)
 648                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 649                                 3 << FM10K_TXINT_TIMER_SHIFT);
 650
 651         /* Setup TX queue */
 652         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 653                 txq = dev->data->tx_queues[i];
 654                 base_addr = txq->hw_ring_phys_addr;
 655                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 656
 657                 /* disable queue to avoid issues while updating state */
 658                 ret = tx_queue_disable(hw, i);
 659                 if (ret) {
 660                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 661                         return -1;
 662                 }
 663                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 664                  * register is read-only for VF.
 665                  */
 666                 if (fm10k_check_ftag(dev->device->devargs)) {
 667                         if (hw->mac.type == fm10k_mac_pf) {
 668                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 669                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 670                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 671                         } else {
 672                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 673                                 return -ENOTSUP;
 674                         }
 675                 }
 676
 677                 /* set location and size for descriptor ring */
 678                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 679                                 base_addr & UINT64_LOWER_32BITS_MASK);
 680                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 681                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 682                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 683
 684                 /* assign default SGLORT for each TX queue by PF */
 685                 if (hw->mac.type == fm10k_mac_pf)
 686                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 687         }
 688
 689         /* set up vector or scalar TX function as appropriate */
 690         fm10k_set_tx_function(dev);
 691
 692         return 0;
 693 }
 694
 695 static int
 696 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 697 {
 698         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 699         struct fm10k_macvlan_filter_info *macvlan;
 700         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 701         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 702         int i, ret;
 703         struct fm10k_rx_queue *rxq;
 704         uint64_t base_addr;
 705         uint32_t size;
 706         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 707         uint32_t logic_port = hw->mac.dglort_map;
 708         uint16_t buf_size;
 709         uint16_t queue_stride = 0;
 710
 711         /* enable RXINT for interrupt mode */
 712         i = 0;
 713         if (rte_intr_dp_is_en(intr_handle)) {
 714                 for (; i < dev->data->nb_rx_queues; i++) {
 715                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 716                         if (hw->mac.type == fm10k_mac_pf)
 717                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 718                                         FM10K_ITR_AUTOMASK |
 719                                         FM10K_ITR_MASK_CLEAR);
 720                         else
 721                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 722                                         FM10K_ITR_AUTOMASK |
 723                                         FM10K_ITR_MASK_CLEAR);
 724                 }
 725         }
 726         /* Disable other RXINT to avoid possible interrupt */
 727         for (; i < hw->mac.max_queues; i++)
 728                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 729                         3 << FM10K_RXINT_TIMER_SHIFT);
 730
 731         /* Setup RX queues */
 732         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 733                 rxq = dev->data->rx_queues[i];
 734                 base_addr = rxq->hw_ring_phys_addr;
 735                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 736
 737                 /* disable queue to avoid issues while updating state */
 738                 ret = rx_queue_disable(hw, i);
 739                 if (ret) {
 740                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 741                         return -1;
 742                 }
 743
 744                 /* Setup the Base and Length of the Rx Descriptor Ring */
 745                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 746                                 base_addr & UINT64_LOWER_32BITS_MASK);
 747                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 748                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 749                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 750
 751                 /* Configure the Rx buffer size for one buff without split */
 752                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 753                         RTE_PKTMBUF_HEADROOM);
 754                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 755                  * reserved for this purpose, and the worst case could be 511B.
 756                  * But SRR reg assumes all buffers have the same size. In order
 757                  * to fill the gap, we'll have to consider the worst case and
 758                  * assume 512B is reserved. If we don't do so, it's possible
 759                  * for HW to overwrite data to next mbuf.
 760                  */
 761                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 762
 763                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 764                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 765                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 766
 767                 /* It adds dual VLAN length for supporting dual VLAN */
 768                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 769                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 770                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 771                         uint32_t reg;
 772                         dev->data->scattered_rx = 1;
 773                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 774                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 775                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 776                 }
 777
 778                 /* Enable drop on empty, it's RO for VF */
 779                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 780                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 781
 782                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 783                 FM10K_WRITE_FLUSH(hw);
 784         }
 785
 786         /* Configure VMDQ/RSS if applicable */
 787         fm10k_dev_mq_rx_configure(dev);
 788
 789         /* Decide the best RX function */
 790         fm10k_set_rx_function(dev);
 791
 792         /* update RX_SGLORT for loopback suppress*/
 793         if (hw->mac.type != fm10k_mac_pf)
 794                 return 0;
 795         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 796         if (macvlan->nb_queue_pools)
 797                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 798         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 799                 if (i && queue_stride && !(i % queue_stride))
 800                         logic_port++;
 801                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 802         }
 803
 804         return 0;
 805 }
 806
 807 static int
 808 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 809 {
 810         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 811         int err = -1;
 812         uint32_t reg;
 813         struct fm10k_rx_queue *rxq;
 814
 815         PMD_INIT_FUNC_TRACE();
 816
 817         if (rx_queue_id < dev->data->nb_rx_queues) {
 818                 rxq = dev->data->rx_queues[rx_queue_id];
 819                 err = rx_queue_reset(rxq);
 820                 if (err == -ENOMEM) {
 821                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 822                         return err;
 823                 } else if (err == -EINVAL) {
 824                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 825                                 " %d", err);
 826                         return err;
 827                 }
 828
 829                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 830                  * Note: this must be done AFTER the queue is enabled on real
 831                  * hardware, but BEFORE the queue is enabled when using the
 832                  * emulation platform. Do it in both places for now and remove
 833                  * this comment and the following two register writes when the
 834                  * emulation platform is no longer being used.
 835                  */
 836                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 837                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 838
 839                 /* Set PF ownership flag for PF devices */
 840                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 841                 if (hw->mac.type == fm10k_mac_pf)
 842                         reg |= FM10K_RXQCTL_PF;
 843                 reg |= FM10K_RXQCTL_ENABLE;
 844                 /* enable RX queue */
 845                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 846                 FM10K_WRITE_FLUSH(hw);
 847
 848                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 849                  * Note: this must be done AFTER the queue is enabled
 850                  */
 851                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 852                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 853                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 854         }
 855
 856         return err;
 857 }
 858
 859 static int
 860 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 861 {
 862         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 863
 864         PMD_INIT_FUNC_TRACE();
 865
 866         if (rx_queue_id < dev->data->nb_rx_queues) {
 867                 /* Disable RX queue */
 868                 rx_queue_disable(hw, rx_queue_id);
 869
 870                 /* Free mbuf and clean HW ring */
 871                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 872                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 873         }
 874
 875         return 0;
 876 }
 877
 878 static int
 879 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 880 {
 881         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 882         /** @todo - this should be defined in the shared code */
 883 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 884         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 885         int err = 0;
 886
 887         PMD_INIT_FUNC_TRACE();
 888
 889         if (tx_queue_id < dev->data->nb_tx_queues) {
 890                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 891
 892                 q->ops->reset(q);
 893
 894                 /* reset head and tail pointers */
 895                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 896                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 897
 898                 /* enable TX queue */
 899                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 900                                         FM10K_TXDCTL_ENABLE | txdctl);
 901                 FM10K_WRITE_FLUSH(hw);
 902                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 903         } else
 904                 err = -1;
 905
 906         return err;
 907 }
 908
 909 static int
 910 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 911 {
 912         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 913
 914         PMD_INIT_FUNC_TRACE();
 915
 916         if (tx_queue_id < dev->data->nb_tx_queues) {
 917                 tx_queue_disable(hw, tx_queue_id);
 918                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 919                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 920         }
 921
 922         return 0;
 923 }
 924
 925 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 926 {
 927         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 928                 != FM10K_DGLORTMAP_NONE);
 929 }
 930
 931 static void
 932 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 933 {
 934         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 935         int status;
 936
 937         PMD_INIT_FUNC_TRACE();
 938
 939         /* Return if it didn't acquire valid glort range */
 940         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 941                 return;
 942
 943         fm10k_mbx_lock(hw);
 944         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 945                                 FM10K_XCAST_MODE_PROMISC);
 946         fm10k_mbx_unlock(hw);
 947
 948         if (status != FM10K_SUCCESS)
 949                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 950 }
 951
 952 static void
 953 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 954 {
 955         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 956         uint8_t mode;
 957         int status;
 958
 959         PMD_INIT_FUNC_TRACE();
 960
 961         /* Return if it didn't acquire valid glort range */
 962         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 963                 return;
 964
 965         if (dev->data->all_multicast == 1)
 966                 mode = FM10K_XCAST_MODE_ALLMULTI;
 967         else
 968                 mode = FM10K_XCAST_MODE_NONE;
 969
 970         fm10k_mbx_lock(hw);
 971         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 972                                 mode);
 973         fm10k_mbx_unlock(hw);
 974
 975         if (status != FM10K_SUCCESS)
 976                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 977 }
 978
 979 static void
 980 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 981 {
 982         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 983         int status;
 984
 985         PMD_INIT_FUNC_TRACE();
 986
 987         /* Return if it didn't acquire valid glort range */
 988         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 989                 return;
 990
 991         /* If promiscuous mode is enabled, it doesn't make sense to enable
 992          * allmulticast and disable promiscuous since fm10k only can select
 993          * one of the modes.
 994          */
 995         if (dev->data->promiscuous) {
 996                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 997                         "needn't enable allmulticast");
 998                 return;
 999         }
1000
1001         fm10k_mbx_lock(hw);
1002         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1003                                 FM10K_XCAST_MODE_ALLMULTI);
1004         fm10k_mbx_unlock(hw);
1005
1006         if (status != FM10K_SUCCESS)
1007                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1008 }
1009
1010 static void
1011 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1012 {
1013         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1014         int status;
1015
1016         PMD_INIT_FUNC_TRACE();
1017
1018         /* Return if it didn't acquire valid glort range */
1019         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1020                 return;
1021
1022         if (dev->data->promiscuous) {
1023                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1024                         "since promisc mode is enabled");
1025                 return;
1026         }
1027
1028         fm10k_mbx_lock(hw);
1029         /* Change mode to unicast mode */
1030         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1031                                 FM10K_XCAST_MODE_NONE);
1032         fm10k_mbx_unlock(hw);
1033
1034         if (status != FM10K_SUCCESS)
1035                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1036 }
1037
1038 static void
1039 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1040 {
1041         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1042         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1043         uint16_t nb_queue_pools;
1044         struct fm10k_macvlan_filter_info *macvlan;
1045
1046         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1047         nb_queue_pools = macvlan->nb_queue_pools;
1048         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1049         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1050
1051         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1052         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1053         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1054                         hw->mac.dglort_map;
1055         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1056         /* Configure VMDQ/RSS DGlort Decoder */
1057         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1058
1059         /* Flow Director configurations, only queue number is valid. */
1060         dglortdec = fls(dev->data->nb_rx_queues - 1);
1061         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1062                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1063         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1064         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1065
1066         /* Invalidate all other GLORT entries */
1067         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1068                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1069                                 FM10K_DGLORTMAP_NONE);
1070 }
1071
1072 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1073 static int
1074 fm10k_dev_start(struct rte_eth_dev *dev)
1075 {
1076         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1077         int i, diag;
1078
1079         PMD_INIT_FUNC_TRACE();
1080
1081         /* stop, init, then start the hw */
1082         diag = fm10k_stop_hw(hw);
1083         if (diag != FM10K_SUCCESS) {
1084                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1085                 return -EIO;
1086         }
1087
1088         diag = fm10k_init_hw(hw);
1089         if (diag != FM10K_SUCCESS) {
1090                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1091                 return -EIO;
1092         }
1093
1094         diag = fm10k_start_hw(hw);
1095         if (diag != FM10K_SUCCESS) {
1096                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1097                 return -EIO;
1098         }
1099
1100         diag = fm10k_dev_tx_init(dev);
1101         if (diag) {
1102                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1103                 return diag;
1104         }
1105
1106         if (fm10k_dev_rxq_interrupt_setup(dev))
1107                 return -EIO;
1108
1109         diag = fm10k_dev_rx_init(dev);
1110         if (diag) {
1111                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1112                 return diag;
1113         }
1114
1115         if (hw->mac.type == fm10k_mac_pf)
1116                 fm10k_dev_dglort_map_configure(dev);
1117
1118         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1119                 struct fm10k_rx_queue *rxq;
1120                 rxq = dev->data->rx_queues[i];
1121
1122                 if (rxq->rx_deferred_start)
1123                         continue;
1124                 diag = fm10k_dev_rx_queue_start(dev, i);
1125                 if (diag != 0) {
1126                         int j;
1127                         for (j = 0; j < i; ++j)
1128                                 rx_queue_clean(dev->data->rx_queues[j]);
1129                         return diag;
1130                 }
1131         }
1132
1133         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1134                 struct fm10k_tx_queue *txq;
1135                 txq = dev->data->tx_queues[i];
1136
1137                 if (txq->tx_deferred_start)
1138                         continue;
1139                 diag = fm10k_dev_tx_queue_start(dev, i);
1140                 if (diag != 0) {
1141                         int j;
1142                         for (j = 0; j < i; ++j)
1143                                 tx_queue_clean(dev->data->tx_queues[j]);
1144                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1145                                 rx_queue_clean(dev->data->rx_queues[j]);
1146                         return diag;
1147                 }
1148         }
1149
1150         /* Update default vlan when not in VMDQ mode */
1151         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1152                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1153
1154         fm10k_link_update(dev, 0);
1155
1156         return 0;
1157 }
1158
1159 static void
1160 fm10k_dev_stop(struct rte_eth_dev *dev)
1161 {
1162         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1163         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1164         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1165         int i;
1166
1167         PMD_INIT_FUNC_TRACE();
1168
1169         if (dev->data->tx_queues)
1170                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1171                         fm10k_dev_tx_queue_stop(dev, i);
1172
1173         if (dev->data->rx_queues)
1174                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1175                         fm10k_dev_rx_queue_stop(dev, i);
1176
1177         /* Disable datapath event */
1178         if (rte_intr_dp_is_en(intr_handle)) {
1179                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1180                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1181                                 3 << FM10K_RXINT_TIMER_SHIFT);
1182                         if (hw->mac.type == fm10k_mac_pf)
1183                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1184                                         FM10K_ITR_MASK_SET);
1185                         else
1186                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1187                                         FM10K_ITR_MASK_SET);
1188                 }
1189         }
1190         /* Clean datapath event and queue/vec mapping */
1191         rte_intr_efd_disable(intr_handle);
1192         rte_free(intr_handle->intr_vec);
1193         intr_handle->intr_vec = NULL;
1194 }
1195
1196 static void
1197 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1198 {
1199         int i;
1200
1201         PMD_INIT_FUNC_TRACE();
1202
1203         if (dev->data->tx_queues) {
1204                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1205                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1206
1207                         tx_queue_free(txq);
1208                 }
1209         }
1210
1211         if (dev->data->rx_queues) {
1212                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1213                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1214         }
1215 }
1216
1217 static void
1218 fm10k_dev_close(struct rte_eth_dev *dev)
1219 {
1220         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1221
1222         PMD_INIT_FUNC_TRACE();
1223
1224         fm10k_mbx_lock(hw);
1225         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1226                 MAX_LPORT_NUM, false);
1227         fm10k_mbx_unlock(hw);
1228
1229         /* allow 100ms for device to quiesce */
1230         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1231
1232         /* Stop mailbox service first */
1233         fm10k_close_mbx_service(hw);
1234         fm10k_dev_stop(dev);
1235         fm10k_dev_queue_release(dev);
1236         fm10k_stop_hw(hw);
1237 }
1238
1239 static int
1240 fm10k_link_update(struct rte_eth_dev *dev,
1241         __rte_unused int wait_to_complete)
1242 {
1243         struct fm10k_dev_info *dev_info =
1244                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1245         PMD_INIT_FUNC_TRACE();
1246
1247         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1248         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1249         dev->data->dev_link.link_status =
1250                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1251         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1252
1253         return 0;
1254 }
1255
1256 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1257         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1258 {
1259         unsigned i, q;
1260         unsigned count = 0;
1261
1262         if (xstats_names != NULL) {
1263                 /* Note: limit checked in rte_eth_xstats_names() */
1264
1265                 /* Global stats */
1266                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1267                         snprintf(xstats_names[count].name,
1268                                 sizeof(xstats_names[count].name),
1269                                 "%s", fm10k_hw_stats_strings[count].name);
1270                         count++;
1271                 }
1272
1273                 /* PF queue stats */
1274                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1275                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1276                                 snprintf(xstats_names[count].name,
1277                                         sizeof(xstats_names[count].name),
1278                                         "rx_q%u_%s", q,
1279                                         fm10k_hw_stats_rx_q_strings[i].name);
1280                                 count++;
1281                         }
1282                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1283                                 snprintf(xstats_names[count].name,
1284                                         sizeof(xstats_names[count].name),
1285                                         "tx_q%u_%s", q,
1286                                         fm10k_hw_stats_tx_q_strings[i].name);
1287                                 count++;
1288                         }
1289                 }
1290         }
1291         return FM10K_NB_XSTATS;
1292 }
1293
1294 static int
1295 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1296                  unsigned n)
1297 {
1298         struct fm10k_hw_stats *hw_stats =
1299                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1300         unsigned i, q, count = 0;
1301
1302         if (n < FM10K_NB_XSTATS)
1303                 return FM10K_NB_XSTATS;
1304
1305         /* Global stats */
1306         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1307                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1308                         fm10k_hw_stats_strings[count].offset);
1309                 xstats[count].id = count;
1310                 count++;
1311         }
1312
1313         /* PF queue stats */
1314         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1315                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1316                         xstats[count].value =
1317                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1318                                 fm10k_hw_stats_rx_q_strings[i].offset);
1319                         xstats[count].id = count;
1320                         count++;
1321                 }
1322                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1323                         xstats[count].value =
1324                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1325                                 fm10k_hw_stats_tx_q_strings[i].offset);
1326                         xstats[count].id = count;
1327                         count++;
1328                 }
1329         }
1330
1331         return FM10K_NB_XSTATS;
1332 }
1333
1334 static int
1335 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1336 {
1337         uint64_t ipackets, opackets, ibytes, obytes;
1338         struct fm10k_hw *hw =
1339                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1340         struct fm10k_hw_stats *hw_stats =
1341                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1342         int i;
1343
1344         PMD_INIT_FUNC_TRACE();
1345
1346         fm10k_update_hw_stats(hw, hw_stats);
1347
1348         ipackets = opackets = ibytes = obytes = 0;
1349         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1350                 (i < hw->mac.max_queues); ++i) {
1351                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1352                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1353                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1354                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1355                 ipackets += stats->q_ipackets[i];
1356                 opackets += stats->q_opackets[i];
1357                 ibytes   += stats->q_ibytes[i];
1358                 obytes   += stats->q_obytes[i];
1359         }
1360         stats->ipackets = ipackets;
1361         stats->opackets = opackets;
1362         stats->ibytes = ibytes;
1363         stats->obytes = obytes;
1364         return 0;
1365 }
1366
1367 static void
1368 fm10k_stats_reset(struct rte_eth_dev *dev)
1369 {
1370         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1371         struct fm10k_hw_stats *hw_stats =
1372                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1373
1374         PMD_INIT_FUNC_TRACE();
1375
1376         memset(hw_stats, 0, sizeof(*hw_stats));
1377         fm10k_rebind_hw_stats(hw, hw_stats);
1378 }
1379
1380 static void
1381 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1382         struct rte_eth_dev_info *dev_info)
1383 {
1384         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1385         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1386
1387         PMD_INIT_FUNC_TRACE();
1388
1389         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1390         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1391         dev_info->max_rx_queues      = hw->mac.max_queues;
1392         dev_info->max_tx_queues      = hw->mac.max_queues;
1393         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1394         dev_info->max_hash_mac_addrs = 0;
1395         dev_info->max_vfs            = pdev->max_vfs;
1396         dev_info->vmdq_pool_base     = 0;
1397         dev_info->vmdq_queue_base    = 0;
1398         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1399         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1400         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1401         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1402                                     dev_info->rx_queue_offload_capa;
1403         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1404         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1405                                     dev_info->tx_queue_offload_capa;
1406
1407         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1408         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1409
1410         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1411                 .rx_thresh = {
1412                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1413                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1414                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1415                 },
1416                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1417                 .rx_drop_en = 0,
1418                 .offloads = 0,
1419         };
1420
1421         dev_info->default_txconf = (struct rte_eth_txconf) {
1422                 .tx_thresh = {
1423                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1424                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1425                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1426                 },
1427                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1428                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1429                 .offloads = 0,
1430         };
1431
1432         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1433                 .nb_max = FM10K_MAX_RX_DESC,
1434                 .nb_min = FM10K_MIN_RX_DESC,
1435                 .nb_align = FM10K_MULT_RX_DESC,
1436         };
1437
1438         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1439                 .nb_max = FM10K_MAX_TX_DESC,
1440                 .nb_min = FM10K_MIN_TX_DESC,
1441                 .nb_align = FM10K_MULT_TX_DESC,
1442                 .nb_seg_max = FM10K_TX_MAX_SEG,
1443                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1444         };
1445
1446         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1447                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1448                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1449 }
1450
1451 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1452 static const uint32_t *
1453 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1454 {
1455         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1456             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1457                 static uint32_t ptypes[] = {
1458                         /* refers to rx_desc_to_ol_flags() */
1459                         RTE_PTYPE_L2_ETHER,
1460                         RTE_PTYPE_L3_IPV4,
1461                         RTE_PTYPE_L3_IPV4_EXT,
1462                         RTE_PTYPE_L3_IPV6,
1463                         RTE_PTYPE_L3_IPV6_EXT,
1464                         RTE_PTYPE_L4_TCP,
1465                         RTE_PTYPE_L4_UDP,
1466                         RTE_PTYPE_UNKNOWN
1467                 };
1468
1469                 return ptypes;
1470         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1471                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1472                 static uint32_t ptypes_vec[] = {
1473                         /* refers to fm10k_desc_to_pktype_v() */
1474                         RTE_PTYPE_L3_IPV4,
1475                         RTE_PTYPE_L3_IPV4_EXT,
1476                         RTE_PTYPE_L3_IPV6,
1477                         RTE_PTYPE_L3_IPV6_EXT,
1478                         RTE_PTYPE_L4_TCP,
1479                         RTE_PTYPE_L4_UDP,
1480                         RTE_PTYPE_TUNNEL_GENEVE,
1481                         RTE_PTYPE_TUNNEL_NVGRE,
1482                         RTE_PTYPE_TUNNEL_VXLAN,
1483                         RTE_PTYPE_TUNNEL_GRE,
1484                         RTE_PTYPE_UNKNOWN
1485                 };
1486
1487                 return ptypes_vec;
1488         }
1489
1490         return NULL;
1491 }
1492 #else
1493 static const uint32_t *
1494 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1495 {
1496         return NULL;
1497 }
1498 #endif
1499
1500 static int
1501 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1502 {
1503         s32 result;
1504         uint16_t mac_num = 0;
1505         uint32_t vid_idx, vid_bit, mac_index;
1506         struct fm10k_hw *hw;
1507         struct fm10k_macvlan_filter_info *macvlan;
1508         struct rte_eth_dev_data *data = dev->data;
1509
1510         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1511         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1512
1513         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1514                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1515                 return -EINVAL;
1516         }
1517
1518         if (vlan_id > ETH_VLAN_ID_MAX) {
1519                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1520                 return -EINVAL;
1521         }
1522
1523         vid_idx = FM10K_VFTA_IDX(vlan_id);
1524         vid_bit = FM10K_VFTA_BIT(vlan_id);
1525         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1526         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1527                 return 0;
1528         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1529         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1530                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1531                         "in the VLAN filter table");
1532                 return -EINVAL;
1533         }
1534
1535         fm10k_mbx_lock(hw);
1536         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1537         fm10k_mbx_unlock(hw);
1538         if (result != FM10K_SUCCESS) {
1539                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1540                 return -EIO;
1541         }
1542
1543         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1544                         (result == FM10K_SUCCESS); mac_index++) {
1545                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1546                         continue;
1547                 if (mac_num > macvlan->mac_num - 1) {
1548                         PMD_INIT_LOG(ERR, "MAC address number "
1549                                         "not match");
1550                         break;
1551                 }
1552                 fm10k_mbx_lock(hw);
1553                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1554                         data->mac_addrs[mac_index].addr_bytes,
1555                         vlan_id, on, 0);
1556                 fm10k_mbx_unlock(hw);
1557                 mac_num++;
1558         }
1559         if (result != FM10K_SUCCESS) {
1560                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1561                 return -EIO;
1562         }
1563
1564         if (on) {
1565                 macvlan->vlan_num++;
1566                 macvlan->vfta[vid_idx] |= vid_bit;
1567         } else {
1568                 macvlan->vlan_num--;
1569                 macvlan->vfta[vid_idx] &= ~vid_bit;
1570         }
1571         return 0;
1572 }
1573
1574 static int
1575 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1576 {
1577         if (mask & ETH_VLAN_STRIP_MASK) {
1578                 if (!(dev->data->dev_conf.rxmode.offloads &
1579                         DEV_RX_OFFLOAD_VLAN_STRIP))
1580                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1581                                         "always on in fm10k");
1582         }
1583
1584         if (mask & ETH_VLAN_EXTEND_MASK) {
1585                 if (dev->data->dev_conf.rxmode.offloads &
1586                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1587                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1588                                         "supported in fm10k");
1589         }
1590
1591         if (mask & ETH_VLAN_FILTER_MASK) {
1592                 if (!(dev->data->dev_conf.rxmode.offloads &
1593                         DEV_RX_OFFLOAD_VLAN_FILTER))
1594                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1595         }
1596
1597         return 0;
1598 }
1599
1600 /* Add/Remove a MAC address, and update filters to main VSI */
1601 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1602                 const u8 *mac, bool add, uint32_t pool)
1603 {
1604         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1605         struct fm10k_macvlan_filter_info *macvlan;
1606         uint32_t i, j, k;
1607
1608         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1609
1610         if (pool != MAIN_VSI_POOL_NUMBER) {
1611                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1612                         "mac to pool %u", pool);
1613                 return;
1614         }
1615         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1616                 if (!macvlan->vfta[j])
1617                         continue;
1618                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1619                         if (!(macvlan->vfta[j] & (1 << k)))
1620                                 continue;
1621                         if (i + 1 > macvlan->vlan_num) {
1622                                 PMD_INIT_LOG(ERR, "vlan number not match");
1623                                 return;
1624                         }
1625                         fm10k_mbx_lock(hw);
1626                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1627                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1628                         fm10k_mbx_unlock(hw);
1629                         i++;
1630                 }
1631         }
1632 }
1633
1634 /* Add/Remove a MAC address, and update filters to VMDQ */
1635 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1636                 const u8 *mac, bool add, uint32_t pool)
1637 {
1638         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1639         struct fm10k_macvlan_filter_info *macvlan;
1640         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1641         uint32_t i;
1642
1643         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1644         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1645
1646         if (pool > macvlan->nb_queue_pools) {
1647                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1648                         " Max pool is %u",
1649                         pool, macvlan->nb_queue_pools);
1650                 return;
1651         }
1652         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1653                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1654                         continue;
1655                 fm10k_mbx_lock(hw);
1656                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1657                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1658                 fm10k_mbx_unlock(hw);
1659         }
1660 }
1661
1662 /* Add/Remove a MAC address, and update filters */
1663 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1664                 const u8 *mac, bool add, uint32_t pool)
1665 {
1666         struct fm10k_macvlan_filter_info *macvlan;
1667
1668         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1669
1670         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1671                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1672         else
1673                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1674
1675         if (add)
1676                 macvlan->mac_num++;
1677         else
1678                 macvlan->mac_num--;
1679 }
1680
1681 /* Add a MAC address, and update filters */
1682 static int
1683 fm10k_macaddr_add(struct rte_eth_dev *dev,
1684                 struct ether_addr *mac_addr,
1685                 uint32_t index,
1686                 uint32_t pool)
1687 {
1688         struct fm10k_macvlan_filter_info *macvlan;
1689
1690         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1691         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1692         macvlan->mac_vmdq_id[index] = pool;
1693         return 0;
1694 }
1695
1696 /* Remove a MAC address, and update filters */
1697 static void
1698 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1699 {
1700         struct rte_eth_dev_data *data = dev->data;
1701         struct fm10k_macvlan_filter_info *macvlan;
1702
1703         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1704         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1705                         FALSE, macvlan->mac_vmdq_id[index]);
1706         macvlan->mac_vmdq_id[index] = 0;
1707 }
1708
1709 static inline int
1710 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1711 {
1712         if ((request < min) || (request > max) || ((request % mult) != 0))
1713                 return -1;
1714         else
1715                 return 0;
1716 }
1717
1718
1719 static inline int
1720 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1721 {
1722         if ((request < min) || (request > max) || ((div % request) != 0))
1723                 return -1;
1724         else
1725                 return 0;
1726 }
1727
1728 static inline int
1729 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1730 {
1731         uint16_t rx_free_thresh;
1732
1733         if (conf->rx_free_thresh == 0)
1734                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1735         else
1736                 rx_free_thresh = conf->rx_free_thresh;
1737
1738         /* make sure the requested threshold satisfies the constraints */
1739         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1740                         FM10K_RX_FREE_THRESH_MAX(q),
1741                         FM10K_RX_FREE_THRESH_DIV(q),
1742                         rx_free_thresh)) {
1743                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1744                         "less than or equal to %u, "
1745                         "greater than or equal to %u, "
1746                         "and a divisor of %u",
1747                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1748                         FM10K_RX_FREE_THRESH_MIN(q),
1749                         FM10K_RX_FREE_THRESH_DIV(q));
1750                 return -EINVAL;
1751         }
1752
1753         q->alloc_thresh = rx_free_thresh;
1754         q->drop_en = conf->rx_drop_en;
1755         q->rx_deferred_start = conf->rx_deferred_start;
1756
1757         return 0;
1758 }
1759
1760 /*
1761  * Hardware requires specific alignment for Rx packet buffers. At
1762  * least one of the following two conditions must be satisfied.
1763  *  1. Address is 512B aligned
1764  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1765  *
1766  * As such, the driver may need to adjust the DMA address within the
1767  * buffer by up to 512B.
1768  *
1769  * return 1 if the element size is valid, otherwise return 0.
1770  */
1771 static int
1772 mempool_element_size_valid(struct rte_mempool *mp)
1773 {
1774         uint32_t min_size;
1775
1776         /* elt_size includes mbuf header and headroom */
1777         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1778                         RTE_PKTMBUF_HEADROOM;
1779
1780         /* account for up to 512B of alignment */
1781         min_size -= FM10K_RX_DATABUF_ALIGN;
1782
1783         /* sanity check for overflow */
1784         if (min_size > mp->elt_size)
1785                 return 0;
1786
1787         /* size is valid */
1788         return 1;
1789 }
1790
1791 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1792 {
1793         RTE_SET_USED(dev);
1794
1795         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1796 }
1797
1798 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1799 {
1800         RTE_SET_USED(dev);
1801
1802         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1803                            DEV_RX_OFFLOAD_VLAN_FILTER |
1804                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1805                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1806                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1807                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1808                            DEV_RX_OFFLOAD_CRC_STRIP   |
1809                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1810 }
1811
1812 static int
1813 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1814         uint16_t nb_desc, unsigned int socket_id,
1815         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1816 {
1817         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1818         struct fm10k_dev_info *dev_info =
1819                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1820         struct fm10k_rx_queue *q;
1821         const struct rte_memzone *mz;
1822         uint64_t offloads;
1823
1824         PMD_INIT_FUNC_TRACE();
1825
1826         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1827
1828         /* make sure the mempool element size can account for alignment. */
1829         if (!mempool_element_size_valid(mp)) {
1830                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1831                 return -EINVAL;
1832         }
1833
1834         /* make sure a valid number of descriptors have been requested */
1835         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1836                                 FM10K_MULT_RX_DESC, nb_desc)) {
1837                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1838                         "less than or equal to %"PRIu32", "
1839                         "greater than or equal to %u, "
1840                         "and a multiple of %u",
1841                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1842                         FM10K_MULT_RX_DESC);
1843                 return -EINVAL;
1844         }
1845
1846         /*
1847          * if this queue existed already, free the associated memory. The
1848          * queue cannot be reused in case we need to allocate memory on
1849          * different socket than was previously used.
1850          */
1851         if (dev->data->rx_queues[queue_id] != NULL) {
1852                 rx_queue_free(dev->data->rx_queues[queue_id]);
1853                 dev->data->rx_queues[queue_id] = NULL;
1854         }
1855
1856         /* allocate memory for the queue structure */
1857         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1858                                 socket_id);
1859         if (q == NULL) {
1860                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1861                 return -ENOMEM;
1862         }
1863
1864         /* setup queue */
1865         q->mp = mp;
1866         q->nb_desc = nb_desc;
1867         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1868         q->port_id = dev->data->port_id;
1869         q->queue_id = queue_id;
1870         q->tail_ptr = (volatile uint32_t *)
1871                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1872         q->offloads = offloads;
1873         if (handle_rxconf(q, conf))
1874                 return -EINVAL;
1875
1876         /* allocate memory for the software ring */
1877         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1878                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1879                         RTE_CACHE_LINE_SIZE, socket_id);
1880         if (q->sw_ring == NULL) {
1881                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1882                 rte_free(q);
1883                 return -ENOMEM;
1884         }
1885
1886         /*
1887          * allocate memory for the hardware descriptor ring. A memzone large
1888          * enough to hold the maximum ring size is requested to allow for
1889          * resizing in later calls to the queue setup function.
1890          */
1891         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1892                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1893                                       socket_id);
1894         if (mz == NULL) {
1895                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1896                 rte_free(q->sw_ring);
1897                 rte_free(q);
1898                 return -ENOMEM;
1899         }
1900         q->hw_ring = mz->addr;
1901         q->hw_ring_phys_addr = mz->iova;
1902
1903         /* Check if number of descs satisfied Vector requirement */
1904         if (!rte_is_power_of_2(nb_desc)) {
1905                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1906                                     "preconditions - canceling the feature for "
1907                                     "the whole port[%d]",
1908                              q->queue_id, q->port_id);
1909                 dev_info->rx_vec_allowed = false;
1910         } else
1911                 fm10k_rxq_vec_setup(q);
1912
1913         dev->data->rx_queues[queue_id] = q;
1914         return 0;
1915 }
1916
1917 static void
1918 fm10k_rx_queue_release(void *queue)
1919 {
1920         PMD_INIT_FUNC_TRACE();
1921
1922         rx_queue_free(queue);
1923 }
1924
1925 static inline int
1926 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1927 {
1928         uint16_t tx_free_thresh;
1929         uint16_t tx_rs_thresh;
1930
1931         /* constraint MACROs require that tx_free_thresh is configured
1932          * before tx_rs_thresh */
1933         if (conf->tx_free_thresh == 0)
1934                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1935         else
1936                 tx_free_thresh = conf->tx_free_thresh;
1937
1938         /* make sure the requested threshold satisfies the constraints */
1939         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1940                         FM10K_TX_FREE_THRESH_MAX(q),
1941                         FM10K_TX_FREE_THRESH_DIV(q),
1942                         tx_free_thresh)) {
1943                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1944                         "less than or equal to %u, "
1945                         "greater than or equal to %u, "
1946                         "and a divisor of %u",
1947                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1948                         FM10K_TX_FREE_THRESH_MIN(q),
1949                         FM10K_TX_FREE_THRESH_DIV(q));
1950                 return -EINVAL;
1951         }
1952
1953         q->free_thresh = tx_free_thresh;
1954
1955         if (conf->tx_rs_thresh == 0)
1956                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1957         else
1958                 tx_rs_thresh = conf->tx_rs_thresh;
1959
1960         q->tx_deferred_start = conf->tx_deferred_start;
1961
1962         /* make sure the requested threshold satisfies the constraints */
1963         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1964                         FM10K_TX_RS_THRESH_MAX(q),
1965                         FM10K_TX_RS_THRESH_DIV(q),
1966                         tx_rs_thresh)) {
1967                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1968                         "less than or equal to %u, "
1969                         "greater than or equal to %u, "
1970                         "and a divisor of %u",
1971                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1972                         FM10K_TX_RS_THRESH_MIN(q),
1973                         FM10K_TX_RS_THRESH_DIV(q));
1974                 return -EINVAL;
1975         }
1976
1977         q->rs_thresh = tx_rs_thresh;
1978
1979         return 0;
1980 }
1981
1982 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
1983 {
1984         RTE_SET_USED(dev);
1985
1986         return 0;
1987 }
1988
1989 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
1990 {
1991         RTE_SET_USED(dev);
1992
1993         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
1994                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
1995                           DEV_TX_OFFLOAD_UDP_CKSUM   |
1996                           DEV_TX_OFFLOAD_TCP_CKSUM   |
1997                           DEV_TX_OFFLOAD_TCP_TSO);
1998 }
1999
2000 static int
2001 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2002         uint16_t nb_desc, unsigned int socket_id,
2003         const struct rte_eth_txconf *conf)
2004 {
2005         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2006         struct fm10k_tx_queue *q;
2007         const struct rte_memzone *mz;
2008         uint64_t offloads;
2009
2010         PMD_INIT_FUNC_TRACE();
2011
2012         offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
2013
2014         /* make sure a valid number of descriptors have been requested */
2015         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2016                                 FM10K_MULT_TX_DESC, nb_desc)) {
2017                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2018                         "less than or equal to %"PRIu32", "
2019                         "greater than or equal to %u, "
2020                         "and a multiple of %u",
2021                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2022                         FM10K_MULT_TX_DESC);
2023                 return -EINVAL;
2024         }
2025
2026         /*
2027          * if this queue existed already, free the associated memory. The
2028          * queue cannot be reused in case we need to allocate memory on
2029          * different socket than was previously used.
2030          */
2031         if (dev->data->tx_queues[queue_id] != NULL) {
2032                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2033
2034                 tx_queue_free(txq);
2035                 dev->data->tx_queues[queue_id] = NULL;
2036         }
2037
2038         /* allocate memory for the queue structure */
2039         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2040                                 socket_id);
2041         if (q == NULL) {
2042                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2043                 return -ENOMEM;
2044         }
2045
2046         /* setup queue */
2047         q->nb_desc = nb_desc;
2048         q->port_id = dev->data->port_id;
2049         q->queue_id = queue_id;
2050         q->offloads = offloads;
2051         q->ops = &def_txq_ops;
2052         q->tail_ptr = (volatile uint32_t *)
2053                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2054         if (handle_txconf(q, conf))
2055                 return -EINVAL;
2056
2057         /* allocate memory for the software ring */
2058         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2059                                         nb_desc * sizeof(struct rte_mbuf *),
2060                                         RTE_CACHE_LINE_SIZE, socket_id);
2061         if (q->sw_ring == NULL) {
2062                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2063                 rte_free(q);
2064                 return -ENOMEM;
2065         }
2066
2067         /*
2068          * allocate memory for the hardware descriptor ring. A memzone large
2069          * enough to hold the maximum ring size is requested to allow for
2070          * resizing in later calls to the queue setup function.
2071          */
2072         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2073                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2074                                       socket_id);
2075         if (mz == NULL) {
2076                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2077                 rte_free(q->sw_ring);
2078                 rte_free(q);
2079                 return -ENOMEM;
2080         }
2081         q->hw_ring = mz->addr;
2082         q->hw_ring_phys_addr = mz->iova;
2083
2084         /*
2085          * allocate memory for the RS bit tracker. Enough slots to hold the
2086          * descriptor index for each RS bit needing to be set are required.
2087          */
2088         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2089                                 ((nb_desc + 1) / q->rs_thresh) *
2090                                 sizeof(uint16_t),
2091                                 RTE_CACHE_LINE_SIZE, socket_id);
2092         if (q->rs_tracker.list == NULL) {
2093                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2094                 rte_free(q->sw_ring);
2095                 rte_free(q);
2096                 return -ENOMEM;
2097         }
2098
2099         dev->data->tx_queues[queue_id] = q;
2100         return 0;
2101 }
2102
2103 static void
2104 fm10k_tx_queue_release(void *queue)
2105 {
2106         struct fm10k_tx_queue *q = queue;
2107         PMD_INIT_FUNC_TRACE();
2108
2109         tx_queue_free(q);
2110 }
2111
2112 static int
2113 fm10k_reta_update(struct rte_eth_dev *dev,
2114                         struct rte_eth_rss_reta_entry64 *reta_conf,
2115                         uint16_t reta_size)
2116 {
2117         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2118         uint16_t i, j, idx, shift;
2119         uint8_t mask;
2120         uint32_t reta;
2121
2122         PMD_INIT_FUNC_TRACE();
2123
2124         if (reta_size > FM10K_MAX_RSS_INDICES) {
2125                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2126                         "(%d) doesn't match the number hardware can supported "
2127                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2128                 return -EINVAL;
2129         }
2130
2131         /*
2132          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2133          * 128-entries in 32 registers
2134          */
2135         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2136                 idx = i / RTE_RETA_GROUP_SIZE;
2137                 shift = i % RTE_RETA_GROUP_SIZE;
2138                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2139                                 BIT_MASK_PER_UINT32);
2140                 if (mask == 0)
2141                         continue;
2142
2143                 reta = 0;
2144                 if (mask != BIT_MASK_PER_UINT32)
2145                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2146
2147                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2148                         if (mask & (0x1 << j)) {
2149                                 if (mask != 0xF)
2150                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2151                                 reta |= reta_conf[idx].reta[shift + j] <<
2152                                                 (CHAR_BIT * j);
2153                         }
2154                 }
2155                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2156         }
2157
2158         return 0;
2159 }
2160
2161 static int
2162 fm10k_reta_query(struct rte_eth_dev *dev,
2163                         struct rte_eth_rss_reta_entry64 *reta_conf,
2164                         uint16_t reta_size)
2165 {
2166         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2167         uint16_t i, j, idx, shift;
2168         uint8_t mask;
2169         uint32_t reta;
2170
2171         PMD_INIT_FUNC_TRACE();
2172
2173         if (reta_size < FM10K_MAX_RSS_INDICES) {
2174                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2175                         "(%d) doesn't match the number hardware can supported "
2176                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2177                 return -EINVAL;
2178         }
2179
2180         /*
2181          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2182          * 128-entries in 32 registers
2183          */
2184         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2185                 idx = i / RTE_RETA_GROUP_SIZE;
2186                 shift = i % RTE_RETA_GROUP_SIZE;
2187                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2188                                 BIT_MASK_PER_UINT32);
2189                 if (mask == 0)
2190                         continue;
2191
2192                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2193                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2194                         if (mask & (0x1 << j))
2195                                 reta_conf[idx].reta[shift + j] = ((reta >>
2196                                         CHAR_BIT * j) & UINT8_MAX);
2197                 }
2198         }
2199
2200         return 0;
2201 }
2202
2203 static int
2204 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2205         struct rte_eth_rss_conf *rss_conf)
2206 {
2207         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2208         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2209         uint32_t mrqc;
2210         uint64_t hf = rss_conf->rss_hf;
2211         int i;
2212
2213         PMD_INIT_FUNC_TRACE();
2214
2215         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2216                                 FM10K_RSSRK_ENTRIES_PER_REG))
2217                 return -EINVAL;
2218
2219         if (hf == 0)
2220                 return -EINVAL;
2221
2222         mrqc = 0;
2223         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2224         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2225         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2226         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2227         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2228         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2229         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2230         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2231         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2232
2233         /* If the mapping doesn't fit any supported, return */
2234         if (mrqc == 0)
2235                 return -EINVAL;
2236
2237         if (key != NULL)
2238                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2239                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2240
2241         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2242
2243         return 0;
2244 }
2245
2246 static int
2247 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2248         struct rte_eth_rss_conf *rss_conf)
2249 {
2250         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2251         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2252         uint32_t mrqc;
2253         uint64_t hf;
2254         int i;
2255
2256         PMD_INIT_FUNC_TRACE();
2257
2258         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2259                                 FM10K_RSSRK_ENTRIES_PER_REG))
2260                 return -EINVAL;
2261
2262         if (key != NULL)
2263                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2264                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2265
2266         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2267         hf = 0;
2268         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2269         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2270         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2271         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2272         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2273         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2274         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2275         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2276         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2277
2278         rss_conf->rss_hf = hf;
2279
2280         return 0;
2281 }
2282
2283 static void
2284 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2285 {
2286         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2287         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2288
2289         /* Bind all local non-queue interrupt to vector 0 */
2290         int_map |= FM10K_MISC_VEC_ID;
2291
2292         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2293         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2294         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2295         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2296         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2297         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2298
2299         /* Enable misc causes */
2300         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2301                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2302                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2303                                 FM10K_EIMR_ENABLE(MAILBOX) |
2304                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2305                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2306                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2307                                 FM10K_EIMR_ENABLE(VFLR));
2308
2309         /* Enable ITR 0 */
2310         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2311                                         FM10K_ITR_MASK_CLEAR);
2312         FM10K_WRITE_FLUSH(hw);
2313 }
2314
2315 static void
2316 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2317 {
2318         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2319         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2320
2321         int_map |= FM10K_MISC_VEC_ID;
2322
2323         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2324         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2325         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2326         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2327         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2328         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2329
2330         /* Disable misc causes */
2331         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2332                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2333                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2334                                 FM10K_EIMR_DISABLE(MAILBOX) |
2335                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2336                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2337                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2338                                 FM10K_EIMR_DISABLE(VFLR));
2339
2340         /* Disable ITR 0 */
2341         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2342         FM10K_WRITE_FLUSH(hw);
2343 }
2344
2345 static void
2346 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2347 {
2348         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2349         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2350
2351         /* Bind all local non-queue interrupt to vector 0 */
2352         int_map |= FM10K_MISC_VEC_ID;
2353
2354         /* Only INT 0 available, other 15 are reserved. */
2355         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2356
2357         /* Enable ITR 0 */
2358         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2359                                         FM10K_ITR_MASK_CLEAR);
2360         FM10K_WRITE_FLUSH(hw);
2361 }
2362
2363 static void
2364 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2365 {
2366         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2367         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2368
2369         int_map |= FM10K_MISC_VEC_ID;
2370
2371         /* Only INT 0 available, other 15 are reserved. */
2372         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2373
2374         /* Disable ITR 0 */
2375         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2376         FM10K_WRITE_FLUSH(hw);
2377 }
2378
2379 static int
2380 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2381 {
2382         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2383         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2384
2385         /* Enable ITR */
2386         if (hw->mac.type == fm10k_mac_pf)
2387                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2388                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2389         else
2390                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2391                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2392         rte_intr_enable(&pdev->intr_handle);
2393         return 0;
2394 }
2395
2396 static int
2397 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2398 {
2399         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2400         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2401
2402         /* Disable ITR */
2403         if (hw->mac.type == fm10k_mac_pf)
2404                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2405                         FM10K_ITR_MASK_SET);
2406         else
2407                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2408                         FM10K_ITR_MASK_SET);
2409         return 0;
2410 }
2411
2412 static int
2413 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2414 {
2415         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2416         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2417         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2418         uint32_t intr_vector, vec;
2419         uint16_t queue_id;
2420         int result = 0;
2421
2422         /* fm10k needs one separate interrupt for mailbox,
2423          * so only drivers which support multiple interrupt vectors
2424          * e.g. vfio-pci can work for fm10k interrupt mode
2425          */
2426         if (!rte_intr_cap_multiple(intr_handle) ||
2427                         dev->data->dev_conf.intr_conf.rxq == 0)
2428                 return result;
2429
2430         intr_vector = dev->data->nb_rx_queues;
2431
2432         /* disable interrupt first */
2433         rte_intr_disable(intr_handle);
2434         if (hw->mac.type == fm10k_mac_pf)
2435                 fm10k_dev_disable_intr_pf(dev);
2436         else
2437                 fm10k_dev_disable_intr_vf(dev);
2438
2439         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2440                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2441                 result = -EIO;
2442         }
2443
2444         if (rte_intr_dp_is_en(intr_handle) && !result) {
2445                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2446                         dev->data->nb_rx_queues * sizeof(int), 0);
2447                 if (intr_handle->intr_vec) {
2448                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2449                                         queue_id < dev->data->nb_rx_queues;
2450                                         queue_id++) {
2451                                 intr_handle->intr_vec[queue_id] = vec;
2452                                 if (vec < intr_handle->nb_efd - 1
2453                                                 + FM10K_RX_VEC_START)
2454                                         vec++;
2455                         }
2456                 } else {
2457                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2458                                 " intr_vec", dev->data->nb_rx_queues);
2459                         rte_intr_efd_disable(intr_handle);
2460                         result = -ENOMEM;
2461                 }
2462         }
2463
2464         if (hw->mac.type == fm10k_mac_pf)
2465                 fm10k_dev_enable_intr_pf(dev);
2466         else
2467                 fm10k_dev_enable_intr_vf(dev);
2468         rte_intr_enable(intr_handle);
2469         hw->mac.ops.update_int_moderator(hw);
2470         return result;
2471 }
2472
2473 static int
2474 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2475 {
2476         struct fm10k_fault fault;
2477         int err;
2478         const char *estr = "Unknown error";
2479
2480         /* Process PCA fault */
2481         if (eicr & FM10K_EICR_PCA_FAULT) {
2482                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2483                 if (err)
2484                         goto error;
2485                 switch (fault.type) {
2486                 case PCA_NO_FAULT:
2487                         estr = "PCA_NO_FAULT"; break;
2488                 case PCA_UNMAPPED_ADDR:
2489                         estr = "PCA_UNMAPPED_ADDR"; break;
2490                 case PCA_BAD_QACCESS_PF:
2491                         estr = "PCA_BAD_QACCESS_PF"; break;
2492                 case PCA_BAD_QACCESS_VF:
2493                         estr = "PCA_BAD_QACCESS_VF"; break;
2494                 case PCA_MALICIOUS_REQ:
2495                         estr = "PCA_MALICIOUS_REQ"; break;
2496                 case PCA_POISONED_TLP:
2497                         estr = "PCA_POISONED_TLP"; break;
2498                 case PCA_TLP_ABORT:
2499                         estr = "PCA_TLP_ABORT"; break;
2500                 default:
2501                         goto error;
2502                 }
2503                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2504                         estr, fault.func ? "VF" : "PF", fault.func,
2505                         fault.address, fault.specinfo);
2506         }
2507
2508         /* Process THI fault */
2509         if (eicr & FM10K_EICR_THI_FAULT) {
2510                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2511                 if (err)
2512                         goto error;
2513                 switch (fault.type) {
2514                 case THI_NO_FAULT:
2515                         estr = "THI_NO_FAULT"; break;
2516                 case THI_MAL_DIS_Q_FAULT:
2517                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2518                 default:
2519                         goto error;
2520                 }
2521                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2522                         estr, fault.func ? "VF" : "PF", fault.func,
2523                         fault.address, fault.specinfo);
2524         }
2525
2526         /* Process FUM fault */
2527         if (eicr & FM10K_EICR_FUM_FAULT) {
2528                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2529                 if (err)
2530                         goto error;
2531                 switch (fault.type) {
2532                 case FUM_NO_FAULT:
2533                         estr = "FUM_NO_FAULT"; break;
2534                 case FUM_UNMAPPED_ADDR:
2535                         estr = "FUM_UNMAPPED_ADDR"; break;
2536                 case FUM_POISONED_TLP:
2537                         estr = "FUM_POISONED_TLP"; break;
2538                 case FUM_BAD_VF_QACCESS:
2539                         estr = "FUM_BAD_VF_QACCESS"; break;
2540                 case FUM_ADD_DECODE_ERR:
2541                         estr = "FUM_ADD_DECODE_ERR"; break;
2542                 case FUM_RO_ERROR:
2543                         estr = "FUM_RO_ERROR"; break;
2544                 case FUM_QPRC_CRC_ERROR:
2545                         estr = "FUM_QPRC_CRC_ERROR"; break;
2546                 case FUM_CSR_TIMEOUT:
2547                         estr = "FUM_CSR_TIMEOUT"; break;
2548                 case FUM_INVALID_TYPE:
2549                         estr = "FUM_INVALID_TYPE"; break;
2550                 case FUM_INVALID_LENGTH:
2551                         estr = "FUM_INVALID_LENGTH"; break;
2552                 case FUM_INVALID_BE:
2553                         estr = "FUM_INVALID_BE"; break;
2554                 case FUM_INVALID_ALIGN:
2555                         estr = "FUM_INVALID_ALIGN"; break;
2556                 default:
2557                         goto error;
2558                 }
2559                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2560                         estr, fault.func ? "VF" : "PF", fault.func,
2561                         fault.address, fault.specinfo);
2562         }
2563
2564         return 0;
2565 error:
2566         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2567         return err;
2568 }
2569
2570 /**
2571  * PF interrupt handler triggered by NIC for handling specific interrupt.
2572  *
2573  * @param handle
2574  *  Pointer to interrupt handle.
2575  * @param param
2576  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2577  *
2578  * @return
2579  *  void
2580  */
2581 static void
2582 fm10k_dev_interrupt_handler_pf(void *param)
2583 {
2584         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2585         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2586         uint32_t cause, status;
2587         struct fm10k_dev_info *dev_info =
2588                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2589         int status_mbx;
2590         s32 err;
2591
2592         if (hw->mac.type != fm10k_mac_pf)
2593                 return;
2594
2595         cause = FM10K_READ_REG(hw, FM10K_EICR);
2596
2597         /* Handle PCI fault cases */
2598         if (cause & FM10K_EICR_FAULT_MASK) {
2599                 PMD_INIT_LOG(ERR, "INT: find fault!");
2600                 fm10k_dev_handle_fault(hw, cause);
2601         }
2602
2603         /* Handle switch up/down */
2604         if (cause & FM10K_EICR_SWITCHNOTREADY)
2605                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2606
2607         if (cause & FM10K_EICR_SWITCHREADY) {
2608                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2609                 if (dev_info->sm_down == 1) {
2610                         fm10k_mbx_lock(hw);
2611
2612                         /* For recreating logical ports */
2613                         status_mbx = hw->mac.ops.update_lport_state(hw,
2614                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2615                         if (status_mbx == FM10K_SUCCESS)
2616                                 PMD_INIT_LOG(INFO,
2617                                         "INT: Recreated Logical port");
2618                         else
2619                                 PMD_INIT_LOG(INFO,
2620                                         "INT: Logical ports weren't recreated");
2621
2622                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2623                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2624                         if (status_mbx != FM10K_SUCCESS)
2625                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2626
2627                         fm10k_mbx_unlock(hw);
2628
2629                         /* first clear the internal SW recording structure */
2630                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2631                                                 ETH_MQ_RX_VMDQ_FLAG))
2632                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2633                                         false);
2634
2635                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2636                                         MAIN_VSI_POOL_NUMBER);
2637
2638                         /*
2639                          * Add default mac address and vlan for the logical
2640                          * ports that have been created, leave to the
2641                          * application to fully recover Rx filtering.
2642                          */
2643                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2644                                         MAIN_VSI_POOL_NUMBER);
2645
2646                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2647                                                 ETH_MQ_RX_VMDQ_FLAG))
2648                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2649                                         true);
2650
2651                         dev_info->sm_down = 0;
2652                         _rte_eth_dev_callback_process(dev,
2653                                         RTE_ETH_EVENT_INTR_LSC,
2654                                         NULL);
2655                 }
2656         }
2657
2658         /* Handle mailbox message */
2659         fm10k_mbx_lock(hw);
2660         err = hw->mbx.ops.process(hw, &hw->mbx);
2661         fm10k_mbx_unlock(hw);
2662
2663         if (err == FM10K_ERR_RESET_REQUESTED) {
2664                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2665                 dev_info->sm_down = 1;
2666                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2667                                 NULL);
2668         }
2669
2670         /* Handle SRAM error */
2671         if (cause & FM10K_EICR_SRAMERROR) {
2672                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2673
2674                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2675                 /* Write to clear pending bits */
2676                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2677
2678                 /* Todo: print out error message after shared code  updates */
2679         }
2680
2681         /* Clear these 3 events if having any */
2682         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2683                  FM10K_EICR_SWITCHREADY;
2684         if (cause)
2685                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2686
2687         /* Re-enable interrupt from device side */
2688         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2689                                         FM10K_ITR_MASK_CLEAR);
2690         /* Re-enable interrupt from host side */
2691         rte_intr_enable(dev->intr_handle);
2692 }
2693
2694 /**
2695  * VF interrupt handler triggered by NIC for handling specific interrupt.
2696  *
2697  * @param handle
2698  *  Pointer to interrupt handle.
2699  * @param param
2700  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2701  *
2702  * @return
2703  *  void
2704  */
2705 static void
2706 fm10k_dev_interrupt_handler_vf(void *param)
2707 {
2708         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2709         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2710         struct fm10k_mbx_info *mbx = &hw->mbx;
2711         struct fm10k_dev_info *dev_info =
2712                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2713         const enum fm10k_mbx_state state = mbx->state;
2714         int status_mbx;
2715
2716         if (hw->mac.type != fm10k_mac_vf)
2717                 return;
2718
2719         /* Handle mailbox message if lock is acquired */
2720         fm10k_mbx_lock(hw);
2721         hw->mbx.ops.process(hw, &hw->mbx);
2722         fm10k_mbx_unlock(hw);
2723
2724         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2725                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2726
2727                 fm10k_mbx_lock(hw);
2728                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2729                                 MAX_LPORT_NUM, 1);
2730                 fm10k_mbx_unlock(hw);
2731
2732                 /* Setting reset flag */
2733                 dev_info->sm_down = 1;
2734                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2735                                 NULL);
2736         }
2737
2738         if (dev_info->sm_down == 1 &&
2739                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2740                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2741                 fm10k_mbx_lock(hw);
2742                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2743                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2744                 if (status_mbx != FM10K_SUCCESS)
2745                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2746                 fm10k_mbx_unlock(hw);
2747
2748                 /* first clear the internal SW recording structure */
2749                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2750                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2751                                 MAIN_VSI_POOL_NUMBER);
2752
2753                 /*
2754                  * Add default mac address and vlan for the logical ports that
2755                  * have been created, leave to the application to fully recover
2756                  * Rx filtering.
2757                  */
2758                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2759                                 MAIN_VSI_POOL_NUMBER);
2760                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2761
2762                 dev_info->sm_down = 0;
2763                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2764                                 NULL);
2765         }
2766
2767         /* Re-enable interrupt from device side */
2768         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2769                                         FM10K_ITR_MASK_CLEAR);
2770         /* Re-enable interrupt from host side */
2771         rte_intr_enable(dev->intr_handle);
2772 }
2773
2774 /* Mailbox message handler in VF */
2775 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2776         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2777         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2778         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2779         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2780 };
2781
2782 static int
2783 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2784 {
2785         int err = 0;
2786
2787         /* Initialize mailbox lock */
2788         fm10k_mbx_initlock(hw);
2789
2790         /* Replace default message handler with new ones */
2791         if (hw->mac.type == fm10k_mac_vf)
2792                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2793
2794         if (err) {
2795                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2796                                 err);
2797                 return err;
2798         }
2799         /* Connect to SM for PF device or PF for VF device */
2800         return hw->mbx.ops.connect(hw, &hw->mbx);
2801 }
2802
2803 static void
2804 fm10k_close_mbx_service(struct fm10k_hw *hw)
2805 {
2806         /* Disconnect from SM for PF device or PF for VF device */
2807         hw->mbx.ops.disconnect(hw, &hw->mbx);
2808 }
2809
2810 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2811         .dev_configure          = fm10k_dev_configure,
2812         .dev_start              = fm10k_dev_start,
2813         .dev_stop               = fm10k_dev_stop,
2814         .dev_close              = fm10k_dev_close,
2815         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2816         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2817         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2818         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2819         .stats_get              = fm10k_stats_get,
2820         .xstats_get             = fm10k_xstats_get,
2821         .xstats_get_names       = fm10k_xstats_get_names,
2822         .stats_reset            = fm10k_stats_reset,
2823         .xstats_reset           = fm10k_stats_reset,
2824         .link_update            = fm10k_link_update,
2825         .dev_infos_get          = fm10k_dev_infos_get,
2826         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2827         .vlan_filter_set        = fm10k_vlan_filter_set,
2828         .vlan_offload_set       = fm10k_vlan_offload_set,
2829         .mac_addr_add           = fm10k_macaddr_add,
2830         .mac_addr_remove        = fm10k_macaddr_remove,
2831         .rx_queue_start         = fm10k_dev_rx_queue_start,
2832         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2833         .tx_queue_start         = fm10k_dev_tx_queue_start,
2834         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2835         .rx_queue_setup         = fm10k_rx_queue_setup,
2836         .rx_queue_release       = fm10k_rx_queue_release,
2837         .tx_queue_setup         = fm10k_tx_queue_setup,
2838         .tx_queue_release       = fm10k_tx_queue_release,
2839         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2840         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2841         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2842         .reta_update            = fm10k_reta_update,
2843         .reta_query             = fm10k_reta_query,
2844         .rss_hash_update        = fm10k_rss_hash_update,
2845         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2846 };
2847
2848 static int ftag_check_handler(__rte_unused const char *key,
2849                 const char *value, __rte_unused void *opaque)
2850 {
2851         if (strcmp(value, "1"))
2852                 return -1;
2853
2854         return 0;
2855 }
2856
2857 static int
2858 fm10k_check_ftag(struct rte_devargs *devargs)
2859 {
2860         struct rte_kvargs *kvlist;
2861         const char *ftag_key = "enable_ftag";
2862
2863         if (devargs == NULL)
2864                 return 0;
2865
2866         kvlist = rte_kvargs_parse(devargs->args, NULL);
2867         if (kvlist == NULL)
2868                 return 0;
2869
2870         if (!rte_kvargs_count(kvlist, ftag_key)) {
2871                 rte_kvargs_free(kvlist);
2872                 return 0;
2873         }
2874         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2875         if (rte_kvargs_process(kvlist, ftag_key,
2876                                 ftag_check_handler, NULL) < 0) {
2877                 rte_kvargs_free(kvlist);
2878                 return 0;
2879         }
2880         rte_kvargs_free(kvlist);
2881
2882         return 1;
2883 }
2884
2885 static uint16_t
2886 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2887                     uint16_t nb_pkts)
2888 {
2889         uint16_t nb_tx = 0;
2890         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2891
2892         while (nb_pkts) {
2893                 uint16_t ret, num;
2894
2895                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2896                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2897                                                  num);
2898                 nb_tx += ret;
2899                 nb_pkts -= ret;
2900                 if (ret < num)
2901                         break;
2902         }
2903
2904         return nb_tx;
2905 }
2906
2907 static void __attribute__((cold))
2908 fm10k_set_tx_function(struct rte_eth_dev *dev)
2909 {
2910         struct fm10k_tx_queue *txq;
2911         int i;
2912         int use_sse = 1;
2913         uint16_t tx_ftag_en = 0;
2914
2915         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2916                 /* primary process has set the ftag flag and offloads */
2917                 txq = dev->data->tx_queues[0];
2918                 if (fm10k_tx_vec_condition_check(txq)) {
2919                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2920                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2921                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2922                 } else {
2923                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2924                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2925                         dev->tx_pkt_prepare = NULL;
2926                 }
2927                 return;
2928         }
2929
2930         if (fm10k_check_ftag(dev->device->devargs))
2931                 tx_ftag_en = 1;
2932
2933         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2934                 txq = dev->data->tx_queues[i];
2935                 txq->tx_ftag_en = tx_ftag_en;
2936                 /* Check if Vector Tx is satisfied */
2937                 if (fm10k_tx_vec_condition_check(txq))
2938                         use_sse = 0;
2939         }
2940
2941         if (use_sse) {
2942                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2943                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2944                         txq = dev->data->tx_queues[i];
2945                         fm10k_txq_vec_setup(txq);
2946                 }
2947                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2948                 dev->tx_pkt_prepare = NULL;
2949         } else {
2950                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2951                 dev->tx_pkt_prepare = fm10k_prep_pkts;
2952                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2953         }
2954 }
2955
2956 static void __attribute__((cold))
2957 fm10k_set_rx_function(struct rte_eth_dev *dev)
2958 {
2959         struct fm10k_dev_info *dev_info =
2960                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2961         uint16_t i, rx_using_sse;
2962         uint16_t rx_ftag_en = 0;
2963
2964         if (fm10k_check_ftag(dev->device->devargs))
2965                 rx_ftag_en = 1;
2966
2967         /* In order to allow Vector Rx there are a few configuration
2968          * conditions to be met.
2969          */
2970         if (!fm10k_rx_vec_condition_check(dev) &&
2971                         dev_info->rx_vec_allowed && !rx_ftag_en) {
2972                 if (dev->data->scattered_rx)
2973                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2974                 else
2975                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2976         } else if (dev->data->scattered_rx)
2977                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2978         else
2979                 dev->rx_pkt_burst = fm10k_recv_pkts;
2980
2981         rx_using_sse =
2982                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2983                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2984
2985         if (rx_using_sse)
2986                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2987         else
2988                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2989
2990         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2991                 return;
2992
2993         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2994                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2995
2996                 rxq->rx_using_sse = rx_using_sse;
2997                 rxq->rx_ftag_en = rx_ftag_en;
2998         }
2999 }
3000
3001 static void
3002 fm10k_params_init(struct rte_eth_dev *dev)
3003 {
3004         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3005         struct fm10k_dev_info *info =
3006                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3007
3008         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3009          * there is no way to get link status without reading BAR4.  Until this
3010          * works, assume we have maximum bandwidth.
3011          * @todo - fix bus info
3012          */
3013         hw->bus_caps.speed = fm10k_bus_speed_8000;
3014         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3015         hw->bus_caps.payload = fm10k_bus_payload_512;
3016         hw->bus.speed = fm10k_bus_speed_8000;
3017         hw->bus.width = fm10k_bus_width_pcie_x8;
3018         hw->bus.payload = fm10k_bus_payload_256;
3019
3020         info->rx_vec_allowed = true;
3021 }
3022
3023 static int
3024 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3025 {
3026         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3027         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3028         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3029         int diag, i;
3030         struct fm10k_macvlan_filter_info *macvlan;
3031
3032         PMD_INIT_FUNC_TRACE();
3033
3034         dev->dev_ops = &fm10k_eth_dev_ops;
3035         dev->rx_pkt_burst = &fm10k_recv_pkts;
3036         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3037         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3038
3039         /*
3040          * Primary process does the whole initialization, for secondary
3041          * processes, we just select the same Rx and Tx function as primary.
3042          */
3043         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3044                 fm10k_set_rx_function(dev);
3045                 fm10k_set_tx_function(dev);
3046                 return 0;
3047         }
3048
3049         rte_eth_copy_pci_info(dev, pdev);
3050
3051         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3052         memset(macvlan, 0, sizeof(*macvlan));
3053         /* Vendor and Device ID need to be set before init of shared code */
3054         memset(hw, 0, sizeof(*hw));
3055         hw->device_id = pdev->id.device_id;
3056         hw->vendor_id = pdev->id.vendor_id;
3057         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3058         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3059         hw->revision_id = 0;
3060         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3061         if (hw->hw_addr == NULL) {
3062                 PMD_INIT_LOG(ERR, "Bad mem resource."
3063                         " Try to blacklist unused devices.");
3064                 return -EIO;
3065         }
3066
3067         /* Store fm10k_adapter pointer */
3068         hw->back = dev->data->dev_private;
3069
3070         /* Initialize the shared code */
3071         diag = fm10k_init_shared_code(hw);
3072         if (diag != FM10K_SUCCESS) {
3073                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3074                 return -EIO;
3075         }
3076
3077         /* Initialize parameters */
3078         fm10k_params_init(dev);
3079
3080         /* Initialize the hw */
3081         diag = fm10k_init_hw(hw);
3082         if (diag != FM10K_SUCCESS) {
3083                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3084                 return -EIO;
3085         }
3086
3087         /* Initialize MAC address(es) */
3088         dev->data->mac_addrs = rte_zmalloc("fm10k",
3089                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3090         if (dev->data->mac_addrs == NULL) {
3091                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3092                 return -ENOMEM;
3093         }
3094
3095         diag = fm10k_read_mac_addr(hw);
3096
3097         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3098                         &dev->data->mac_addrs[0]);
3099
3100         if (diag != FM10K_SUCCESS ||
3101                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3102
3103                 /* Generate a random addr */
3104                 eth_random_addr(hw->mac.addr);
3105                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3106                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3107                 &dev->data->mac_addrs[0]);
3108         }
3109
3110         /* Reset the hw statistics */
3111         fm10k_stats_reset(dev);
3112
3113         /* Reset the hw */
3114         diag = fm10k_reset_hw(hw);
3115         if (diag != FM10K_SUCCESS) {
3116                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3117                 return -EIO;
3118         }
3119
3120         /* Setup mailbox service */
3121         diag = fm10k_setup_mbx_service(hw);
3122         if (diag != FM10K_SUCCESS) {
3123                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3124                 return -EIO;
3125         }
3126
3127         /*PF/VF has different interrupt handling mechanism */
3128         if (hw->mac.type == fm10k_mac_pf) {
3129                 /* register callback func to eal lib */
3130                 rte_intr_callback_register(intr_handle,
3131                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3132
3133                 /* enable MISC interrupt */
3134                 fm10k_dev_enable_intr_pf(dev);
3135         } else { /* VF */
3136                 rte_intr_callback_register(intr_handle,
3137                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3138
3139                 fm10k_dev_enable_intr_vf(dev);
3140         }
3141
3142         /* Enable intr after callback registered */
3143         rte_intr_enable(intr_handle);
3144
3145         hw->mac.ops.update_int_moderator(hw);
3146
3147         /* Make sure Switch Manager is ready before going forward. */
3148         if (hw->mac.type == fm10k_mac_pf) {
3149                 int switch_ready = 0;
3150
3151                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3152                         fm10k_mbx_lock(hw);
3153                         hw->mac.ops.get_host_state(hw, &switch_ready);
3154                         fm10k_mbx_unlock(hw);
3155                         if (switch_ready)
3156                                 break;
3157                         /* Delay some time to acquire async LPORT_MAP info. */
3158                         rte_delay_us(WAIT_SWITCH_MSG_US);
3159                 }
3160
3161                 if (switch_ready == 0) {
3162                         PMD_INIT_LOG(ERR, "switch is not ready");
3163                         return -1;
3164                 }
3165         }
3166
3167         /*
3168          * Below function will trigger operations on mailbox, acquire lock to
3169          * avoid race condition from interrupt handler. Operations on mailbox
3170          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3171          * will handle and generate an interrupt to our side. Then,  FIFO in
3172          * mailbox will be touched.
3173          */
3174         fm10k_mbx_lock(hw);
3175         /* Enable port first */
3176         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3177                                         MAX_LPORT_NUM, 1);
3178
3179         /* Set unicast mode by default. App can change to other mode in other
3180          * API func.
3181          */
3182         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3183                                         FM10K_XCAST_MODE_NONE);
3184
3185         fm10k_mbx_unlock(hw);
3186
3187         /* Make sure default VID is ready before going forward. */
3188         if (hw->mac.type == fm10k_mac_pf) {
3189                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3190                         if (hw->mac.default_vid)
3191                                 break;
3192                         /* Delay some time to acquire async port VLAN info. */
3193                         rte_delay_us(WAIT_SWITCH_MSG_US);
3194                 }
3195
3196                 if (!hw->mac.default_vid) {
3197                         PMD_INIT_LOG(ERR, "default VID is not ready");
3198                         return -1;
3199                 }
3200         }
3201
3202         /* Add default mac address */
3203         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3204                 MAIN_VSI_POOL_NUMBER);
3205
3206         return 0;
3207 }
3208
3209 static int
3210 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3211 {
3212         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3213         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3214         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3215         PMD_INIT_FUNC_TRACE();
3216
3217         /* only uninitialize in the primary process */
3218         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3219                 return 0;
3220
3221         /* safe to close dev here */
3222         fm10k_dev_close(dev);
3223
3224         dev->dev_ops = NULL;
3225         dev->rx_pkt_burst = NULL;
3226         dev->tx_pkt_burst = NULL;
3227
3228         /* disable uio/vfio intr */
3229         rte_intr_disable(intr_handle);
3230
3231         /*PF/VF has different interrupt handling mechanism */
3232         if (hw->mac.type == fm10k_mac_pf) {
3233                 /* disable interrupt */
3234                 fm10k_dev_disable_intr_pf(dev);
3235
3236                 /* unregister callback func to eal lib */
3237                 rte_intr_callback_unregister(intr_handle,
3238                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3239         } else {
3240                 /* disable interrupt */
3241                 fm10k_dev_disable_intr_vf(dev);
3242
3243                 rte_intr_callback_unregister(intr_handle,
3244                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3245         }
3246
3247         /* free mac memory */
3248         if (dev->data->mac_addrs) {
3249                 rte_free(dev->data->mac_addrs);
3250                 dev->data->mac_addrs = NULL;
3251         }
3252
3253         memset(hw, 0, sizeof(*hw));
3254
3255         return 0;
3256 }
3257
3258 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3259         struct rte_pci_device *pci_dev)
3260 {
3261         return rte_eth_dev_pci_generic_probe(pci_dev,
3262                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3263 }
3264
3265 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3266 {
3267         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3268 }
3269
3270 /*
3271  * The set of PCI devices this driver supports. This driver will enable both PF
3272  * and SRIOV-VF devices.
3273  */
3274 static const struct rte_pci_id pci_id_fm10k_map[] = {
3275         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3276         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3277         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3278         { .vendor_id = 0, /* sentinel */ },
3279 };
3280
3281 static struct rte_pci_driver rte_pmd_fm10k = {
3282         .id_table = pci_id_fm10k_map,
3283         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3284                      RTE_PCI_DRV_IOVA_AS_VA,
3285         .probe = eth_fm10k_pci_probe,
3286         .remove = eth_fm10k_pci_remove,
3287 };
3288
3289 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3290 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3291 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3292
3293 RTE_INIT(fm10k_init_log);
3294 static void
3295 fm10k_init_log(void)
3296 {
3297         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3298         if (fm10k_logtype_init >= 0)
3299                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3300         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3301         if (fm10k_logtype_driver >= 0)
3302                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3303 }