drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451         struct rte_eth_dev_info dev_info;
 452         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
 453         uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
 454
 455         PMD_INIT_FUNC_TRACE();
 456
 457         if ((rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP) == 0)
 458                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 459
 460         fm10k_dev_infos_get(dev, &dev_info);
 461         if ((rx_offloads & dev_info.rx_offload_capa) != rx_offloads) {
 462                 PMD_DRV_LOG(ERR, "Some Rx offloads are not supported "
 463                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 464                             rx_offloads, dev_info.rx_offload_capa);
 465                 return -ENOTSUP;
 466         }
 467         if ((tx_offloads & dev_info.tx_offload_capa) != tx_offloads) {
 468                 PMD_DRV_LOG(ERR, "Some Tx offloads are not supported "
 469                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 470                             tx_offloads, dev_info.tx_offload_capa);
 471                 return -ENOTSUP;
 472         }
 473
 474         /* multipe queue mode checking */
 475         ret  = fm10k_check_mq_mode(dev);
 476         if (ret != 0) {
 477                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 478                             ret);
 479                 return ret;
 480         }
 481
 482         dev->data->scattered_rx = 0;
 483
 484         return 0;
 485 }
 486
 487 /* fls = find last set bit = 32 minus the number of leading zeros */
 488 #ifndef fls
 489 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 490 #endif
 491
 492 static void
 493 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 494 {
 495         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 496         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 497         uint32_t i;
 498
 499         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 500
 501         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 502                 if (!vmdq_conf->pool_map[i].pools)
 503                         continue;
 504                 fm10k_mbx_lock(hw);
 505                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 506                 fm10k_mbx_unlock(hw);
 507         }
 508 }
 509
 510 static void
 511 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 512 {
 513         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 514
 515         /* Add default mac address */
 516         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 517                 MAIN_VSI_POOL_NUMBER);
 518 }
 519
 520 static void
 521 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 522 {
 523         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 524         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 525         uint32_t mrqc, *key, i, reta, j;
 526         uint64_t hf;
 527
 528 #define RSS_KEY_SIZE 40
 529         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 530                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 531                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 532                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 533                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 534                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 535         };
 536
 537         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 538                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 539                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 540                 return;
 541         }
 542
 543         /* random key is rss_intel_key (default) or user provided (rss_key) */
 544         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 545                 key = (uint32_t *)rss_intel_key;
 546         else
 547                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 548
 549         /* Now fill our hash function seeds, 4 bytes at a time */
 550         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 551                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 552
 553         /*
 554          * Fill in redirection table
 555          * The byte-swap is needed because NIC registers are in
 556          * little-endian order.
 557          */
 558         reta = 0;
 559         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 560                 if (j == dev->data->nb_rx_queues)
 561                         j = 0;
 562                 reta = (reta << CHAR_BIT) | j;
 563                 if ((i & 3) == 3)
 564                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 565                                         rte_bswap32(reta));
 566         }
 567
 568         /*
 569          * Generate RSS hash based on packet types, TCP/UDP
 570          * port numbers and/or IPv4/v6 src and dst addresses
 571          */
 572         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 573         mrqc = 0;
 574         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 578         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 581         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 582         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 583
 584         if (mrqc == 0) {
 585                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 586                         "supported", hf);
 587                 return;
 588         }
 589
 590         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 591 }
 592
 593 static void
 594 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 595 {
 596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 597         uint32_t i;
 598
 599         for (i = 0; i < nb_lport_new; i++) {
 600                 /* Set unicast mode by default. App can change
 601                  * to other mode in other API func.
 602                  */
 603                 fm10k_mbx_lock(hw);
 604                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 605                         FM10K_XCAST_MODE_NONE);
 606                 fm10k_mbx_unlock(hw);
 607         }
 608 }
 609
 610 static void
 611 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 612 {
 613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 615         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 616         struct fm10k_macvlan_filter_info *macvlan;
 617         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 618         uint16_t nb_lport_new;
 619
 620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 621         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 622
 623         fm10k_dev_rss_configure(dev);
 624
 625         /* only PF supports VMDQ */
 626         if (hw->mac.type != fm10k_mac_pf)
 627                 return;
 628
 629         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 630                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 631
 632         /* no pool number change, no need to update logic port and VLAN/MAC */
 633         if (macvlan->nb_queue_pools == nb_queue_pools)
 634                 return;
 635
 636         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 637         fm10k_dev_logic_port_update(dev, nb_lport_new);
 638
 639         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 640         memset(dev->data->mac_addrs, 0,
 641                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 642         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 643                 &dev->data->mac_addrs[0]);
 644         memset(macvlan, 0, sizeof(*macvlan));
 645         macvlan->nb_queue_pools = nb_queue_pools;
 646
 647         if (nb_queue_pools)
 648                 fm10k_dev_vmdq_rx_configure(dev);
 649         else
 650                 fm10k_dev_pf_main_vsi_reset(dev);
 651 }
 652
 653 static int
 654 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 655 {
 656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 657         int i, ret;
 658         struct fm10k_tx_queue *txq;
 659         uint64_t base_addr;
 660         uint32_t size;
 661
 662         /* Disable TXINT to avoid possible interrupt */
 663         for (i = 0; i < hw->mac.max_queues; i++)
 664                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 665                                 3 << FM10K_TXINT_TIMER_SHIFT);
 666
 667         /* Setup TX queue */
 668         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 669                 txq = dev->data->tx_queues[i];
 670                 base_addr = txq->hw_ring_phys_addr;
 671                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 672
 673                 /* disable queue to avoid issues while updating state */
 674                 ret = tx_queue_disable(hw, i);
 675                 if (ret) {
 676                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 677                         return -1;
 678                 }
 679                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 680                  * register is read-only for VF.
 681                  */
 682                 if (fm10k_check_ftag(dev->device->devargs)) {
 683                         if (hw->mac.type == fm10k_mac_pf) {
 684                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 685                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 686                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 687                         } else {
 688                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 689                                 return -ENOTSUP;
 690                         }
 691                 }
 692
 693                 /* set location and size for descriptor ring */
 694                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 695                                 base_addr & UINT64_LOWER_32BITS_MASK);
 696                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 697                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 698                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 699
 700                 /* assign default SGLORT for each TX queue by PF */
 701                 if (hw->mac.type == fm10k_mac_pf)
 702                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 703         }
 704
 705         /* set up vector or scalar TX function as appropriate */
 706         fm10k_set_tx_function(dev);
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         struct fm10k_macvlan_filter_info *macvlan;
 716         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 717         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 718         int i, ret;
 719         struct fm10k_rx_queue *rxq;
 720         uint64_t base_addr;
 721         uint32_t size;
 722         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 723         uint32_t logic_port = hw->mac.dglort_map;
 724         uint16_t buf_size;
 725         uint16_t queue_stride = 0;
 726
 727         /* enable RXINT for interrupt mode */
 728         i = 0;
 729         if (rte_intr_dp_is_en(intr_handle)) {
 730                 for (; i < dev->data->nb_rx_queues; i++) {
 731                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 732                         if (hw->mac.type == fm10k_mac_pf)
 733                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 734                                         FM10K_ITR_AUTOMASK |
 735                                         FM10K_ITR_MASK_CLEAR);
 736                         else
 737                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 738                                         FM10K_ITR_AUTOMASK |
 739                                         FM10K_ITR_MASK_CLEAR);
 740                 }
 741         }
 742         /* Disable other RXINT to avoid possible interrupt */
 743         for (; i < hw->mac.max_queues; i++)
 744                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 745                         3 << FM10K_RXINT_TIMER_SHIFT);
 746
 747         /* Setup RX queues */
 748         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 749                 rxq = dev->data->rx_queues[i];
 750                 base_addr = rxq->hw_ring_phys_addr;
 751                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 752
 753                 /* disable queue to avoid issues while updating state */
 754                 ret = rx_queue_disable(hw, i);
 755                 if (ret) {
 756                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 757                         return -1;
 758                 }
 759
 760                 /* Setup the Base and Length of the Rx Descriptor Ring */
 761                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 762                                 base_addr & UINT64_LOWER_32BITS_MASK);
 763                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 764                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 765                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 766
 767                 /* Configure the Rx buffer size for one buff without split */
 768                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 769                         RTE_PKTMBUF_HEADROOM);
 770                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 771                  * reserved for this purpose, and the worst case could be 511B.
 772                  * But SRR reg assumes all buffers have the same size. In order
 773                  * to fill the gap, we'll have to consider the worst case and
 774                  * assume 512B is reserved. If we don't do so, it's possible
 775                  * for HW to overwrite data to next mbuf.
 776                  */
 777                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 778
 779                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 780                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 781                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 782
 783                 /* It adds dual VLAN length for supporting dual VLAN */
 784                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 785                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 786                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 787                         uint32_t reg;
 788                         dev->data->scattered_rx = 1;
 789                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 790                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 791                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 792                 }
 793
 794                 /* Enable drop on empty, it's RO for VF */
 795                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 796                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 797
 798                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 799                 FM10K_WRITE_FLUSH(hw);
 800         }
 801
 802         /* Configure VMDQ/RSS if applicable */
 803         fm10k_dev_mq_rx_configure(dev);
 804
 805         /* Decide the best RX function */
 806         fm10k_set_rx_function(dev);
 807
 808         /* update RX_SGLORT for loopback suppress*/
 809         if (hw->mac.type != fm10k_mac_pf)
 810                 return 0;
 811         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 812         if (macvlan->nb_queue_pools)
 813                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 814         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 815                 if (i && queue_stride && !(i % queue_stride))
 816                         logic_port++;
 817                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 818         }
 819
 820         return 0;
 821 }
 822
 823 static int
 824 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 825 {
 826         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 827         int err = -1;
 828         uint32_t reg;
 829         struct fm10k_rx_queue *rxq;
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         if (rx_queue_id < dev->data->nb_rx_queues) {
 834                 rxq = dev->data->rx_queues[rx_queue_id];
 835                 err = rx_queue_reset(rxq);
 836                 if (err == -ENOMEM) {
 837                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 838                         return err;
 839                 } else if (err == -EINVAL) {
 840                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 841                                 " %d", err);
 842                         return err;
 843                 }
 844
 845                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 846                  * Note: this must be done AFTER the queue is enabled on real
 847                  * hardware, but BEFORE the queue is enabled when using the
 848                  * emulation platform. Do it in both places for now and remove
 849                  * this comment and the following two register writes when the
 850                  * emulation platform is no longer being used.
 851                  */
 852                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854
 855                 /* Set PF ownership flag for PF devices */
 856                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 857                 if (hw->mac.type == fm10k_mac_pf)
 858                         reg |= FM10K_RXQCTL_PF;
 859                 reg |= FM10K_RXQCTL_ENABLE;
 860                 /* enable RX queue */
 861                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 862                 FM10K_WRITE_FLUSH(hw);
 863
 864                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 865                  * Note: this must be done AFTER the queue is enabled
 866                  */
 867                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 868                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 869                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 870         }
 871
 872         return err;
 873 }
 874
 875 static int
 876 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (rx_queue_id < dev->data->nb_rx_queues) {
 883                 /* Disable RX queue */
 884                 rx_queue_disable(hw, rx_queue_id);
 885
 886                 /* Free mbuf and clean HW ring */
 887                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 888                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 896 {
 897         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         /** @todo - this should be defined in the shared code */
 899 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 900         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 901         int err = 0;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (tx_queue_id < dev->data->nb_tx_queues) {
 906                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 907
 908                 q->ops->reset(q);
 909
 910                 /* reset head and tail pointers */
 911                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 912                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 913
 914                 /* enable TX queue */
 915                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 916                                         FM10K_TXDCTL_ENABLE | txdctl);
 917                 FM10K_WRITE_FLUSH(hw);
 918                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 919         } else
 920                 err = -1;
 921
 922         return err;
 923 }
 924
 925 static int
 926 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 927 {
 928         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (tx_queue_id < dev->data->nb_tx_queues) {
 933                 tx_queue_disable(hw, tx_queue_id);
 934                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 935                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 936         }
 937
 938         return 0;
 939 }
 940
 941 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 942 {
 943         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 944                 != FM10K_DGLORTMAP_NONE);
 945 }
 946
 947 static void
 948 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 949 {
 950         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return;
 958
 959         fm10k_mbx_lock(hw);
 960         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 961                                 FM10K_XCAST_MODE_PROMISC);
 962         fm10k_mbx_unlock(hw);
 963
 964         if (status != FM10K_SUCCESS)
 965                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 966 }
 967
 968 static void
 969 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint8_t mode;
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return;
 980
 981         if (dev->data->all_multicast == 1)
 982                 mode = FM10K_XCAST_MODE_ALLMULTI;
 983         else
 984                 mode = FM10K_XCAST_MODE_NONE;
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 mode);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         /* If promiscuous mode is enabled, it doesn't make sense to enable
1008          * allmulticast and disable promiscuous since fm10k only can select
1009          * one of the modes.
1010          */
1011         if (dev->data->promiscuous) {
1012                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1013                         "needn't enable allmulticast");
1014                 return;
1015         }
1016
1017         fm10k_mbx_lock(hw);
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_ALLMULTI);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         int status;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Return if it didn't acquire valid glort range */
1035         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1036                 return;
1037
1038         if (dev->data->promiscuous) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1040                         "since promisc mode is enabled");
1041                 return;
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         /* Change mode to unicast mode */
1046         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1047                                 FM10K_XCAST_MODE_NONE);
1048         fm10k_mbx_unlock(hw);
1049
1050         if (status != FM10K_SUCCESS)
1051                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1052 }
1053
1054 static void
1055 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1056 {
1057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1059         uint16_t nb_queue_pools;
1060         struct fm10k_macvlan_filter_info *macvlan;
1061
1062         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1063         nb_queue_pools = macvlan->nb_queue_pools;
1064         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1065         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1066
1067         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1068         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1069         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         hw->mac.dglort_map;
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1072         /* Configure VMDQ/RSS DGlort Decoder */
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1074
1075         /* Flow Director configurations, only queue number is valid. */
1076         dglortdec = fls(dev->data->nb_rx_queues - 1);
1077         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1078                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1080         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1081
1082         /* Invalidate all other GLORT entries */
1083         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1084                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1085                                 FM10K_DGLORTMAP_NONE);
1086 }
1087
1088 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1089 static int
1090 fm10k_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         int i, diag;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* stop, init, then start the hw */
1098         diag = fm10k_stop_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_init_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_start_hw(hw);
1111         if (diag != FM10K_SUCCESS) {
1112                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1113                 return -EIO;
1114         }
1115
1116         diag = fm10k_dev_tx_init(dev);
1117         if (diag) {
1118                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1119                 return diag;
1120         }
1121
1122         if (fm10k_dev_rxq_interrupt_setup(dev))
1123                 return -EIO;
1124
1125         diag = fm10k_dev_rx_init(dev);
1126         if (diag) {
1127                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1128                 return diag;
1129         }
1130
1131         if (hw->mac.type == fm10k_mac_pf)
1132                 fm10k_dev_dglort_map_configure(dev);
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 struct fm10k_rx_queue *rxq;
1136                 rxq = dev->data->rx_queues[i];
1137
1138                 if (rxq->rx_deferred_start)
1139                         continue;
1140                 diag = fm10k_dev_rx_queue_start(dev, i);
1141                 if (diag != 0) {
1142                         int j;
1143                         for (j = 0; j < i; ++j)
1144                                 rx_queue_clean(dev->data->rx_queues[j]);
1145                         return diag;
1146                 }
1147         }
1148
1149         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1150                 struct fm10k_tx_queue *txq;
1151                 txq = dev->data->tx_queues[i];
1152
1153                 if (txq->tx_deferred_start)
1154                         continue;
1155                 diag = fm10k_dev_tx_queue_start(dev, i);
1156                 if (diag != 0) {
1157                         int j;
1158                         for (j = 0; j < i; ++j)
1159                                 tx_queue_clean(dev->data->tx_queues[j]);
1160                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1161                                 rx_queue_clean(dev->data->rx_queues[j]);
1162                         return diag;
1163                 }
1164         }
1165
1166         /* Update default vlan when not in VMDQ mode */
1167         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1168                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1169
1170         fm10k_link_update(dev, 0);
1171
1172         return 0;
1173 }
1174
1175 static void
1176 fm10k_dev_stop(struct rte_eth_dev *dev)
1177 {
1178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1179         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1180         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1181         int i;
1182
1183         PMD_INIT_FUNC_TRACE();
1184
1185         if (dev->data->tx_queues)
1186                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1187                         fm10k_dev_tx_queue_stop(dev, i);
1188
1189         if (dev->data->rx_queues)
1190                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1191                         fm10k_dev_rx_queue_stop(dev, i);
1192
1193         /* Disable datapath event */
1194         if (rte_intr_dp_is_en(intr_handle)) {
1195                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1196                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1197                                 3 << FM10K_RXINT_TIMER_SHIFT);
1198                         if (hw->mac.type == fm10k_mac_pf)
1199                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                         else
1202                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1203                                         FM10K_ITR_MASK_SET);
1204                 }
1205         }
1206         /* Clean datapath event and queue/vec mapping */
1207         rte_intr_efd_disable(intr_handle);
1208         rte_free(intr_handle->intr_vec);
1209         intr_handle->intr_vec = NULL;
1210 }
1211
1212 static void
1213 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1214 {
1215         int i;
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         if (dev->data->tx_queues) {
1220                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1222
1223                         tx_queue_free(txq);
1224                 }
1225         }
1226
1227         if (dev->data->rx_queues) {
1228                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1229                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1230         }
1231 }
1232
1233 static void
1234 fm10k_dev_close(struct rte_eth_dev *dev)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237
1238         PMD_INIT_FUNC_TRACE();
1239
1240         fm10k_mbx_lock(hw);
1241         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1242                 MAX_LPORT_NUM, false);
1243         fm10k_mbx_unlock(hw);
1244
1245         /* allow 100ms for device to quiesce */
1246         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1247
1248         /* Stop mailbox service first */
1249         fm10k_close_mbx_service(hw);
1250         fm10k_dev_stop(dev);
1251         fm10k_dev_queue_release(dev);
1252         fm10k_stop_hw(hw);
1253 }
1254
1255 static int
1256 fm10k_link_update(struct rte_eth_dev *dev,
1257         __rte_unused int wait_to_complete)
1258 {
1259         struct fm10k_dev_info *dev_info =
1260                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1261         PMD_INIT_FUNC_TRACE();
1262
1263         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1264         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1265         dev->data->dev_link.link_status =
1266                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1267         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1268
1269         return 0;
1270 }
1271
1272 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1273         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1274 {
1275         unsigned i, q;
1276         unsigned count = 0;
1277
1278         if (xstats_names != NULL) {
1279                 /* Note: limit checked in rte_eth_xstats_names() */
1280
1281                 /* Global stats */
1282                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1283                         snprintf(xstats_names[count].name,
1284                                 sizeof(xstats_names[count].name),
1285                                 "%s", fm10k_hw_stats_strings[count].name);
1286                         count++;
1287                 }
1288
1289                 /* PF queue stats */
1290                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1291                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1292                                 snprintf(xstats_names[count].name,
1293                                         sizeof(xstats_names[count].name),
1294                                         "rx_q%u_%s", q,
1295                                         fm10k_hw_stats_rx_q_strings[i].name);
1296                                 count++;
1297                         }
1298                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1299                                 snprintf(xstats_names[count].name,
1300                                         sizeof(xstats_names[count].name),
1301                                         "tx_q%u_%s", q,
1302                                         fm10k_hw_stats_tx_q_strings[i].name);
1303                                 count++;
1304                         }
1305                 }
1306         }
1307         return FM10K_NB_XSTATS;
1308 }
1309
1310 static int
1311 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1312                  unsigned n)
1313 {
1314         struct fm10k_hw_stats *hw_stats =
1315                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1316         unsigned i, q, count = 0;
1317
1318         if (n < FM10K_NB_XSTATS)
1319                 return FM10K_NB_XSTATS;
1320
1321         /* Global stats */
1322         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1323                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1324                         fm10k_hw_stats_strings[count].offset);
1325                 xstats[count].id = count;
1326                 count++;
1327         }
1328
1329         /* PF queue stats */
1330         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1331                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1332                         xstats[count].value =
1333                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1334                                 fm10k_hw_stats_rx_q_strings[i].offset);
1335                         xstats[count].id = count;
1336                         count++;
1337                 }
1338                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1339                         xstats[count].value =
1340                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1341                                 fm10k_hw_stats_tx_q_strings[i].offset);
1342                         xstats[count].id = count;
1343                         count++;
1344                 }
1345         }
1346
1347         return FM10K_NB_XSTATS;
1348 }
1349
1350 static int
1351 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1352 {
1353         uint64_t ipackets, opackets, ibytes, obytes;
1354         struct fm10k_hw *hw =
1355                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1356         struct fm10k_hw_stats *hw_stats =
1357                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1358         int i;
1359
1360         PMD_INIT_FUNC_TRACE();
1361
1362         fm10k_update_hw_stats(hw, hw_stats);
1363
1364         ipackets = opackets = ibytes = obytes = 0;
1365         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1366                 (i < hw->mac.max_queues); ++i) {
1367                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1368                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1369                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1370                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1371                 ipackets += stats->q_ipackets[i];
1372                 opackets += stats->q_opackets[i];
1373                 ibytes   += stats->q_ibytes[i];
1374                 obytes   += stats->q_obytes[i];
1375         }
1376         stats->ipackets = ipackets;
1377         stats->opackets = opackets;
1378         stats->ibytes = ibytes;
1379         stats->obytes = obytes;
1380         return 0;
1381 }
1382
1383 static void
1384 fm10k_stats_reset(struct rte_eth_dev *dev)
1385 {
1386         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1387         struct fm10k_hw_stats *hw_stats =
1388                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1389
1390         PMD_INIT_FUNC_TRACE();
1391
1392         memset(hw_stats, 0, sizeof(*hw_stats));
1393         fm10k_rebind_hw_stats(hw, hw_stats);
1394 }
1395
1396 static void
1397 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1398         struct rte_eth_dev_info *dev_info)
1399 {
1400         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1401         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1402
1403         PMD_INIT_FUNC_TRACE();
1404
1405         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1406         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1407         dev_info->max_rx_queues      = hw->mac.max_queues;
1408         dev_info->max_tx_queues      = hw->mac.max_queues;
1409         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1410         dev_info->max_hash_mac_addrs = 0;
1411         dev_info->max_vfs            = pdev->max_vfs;
1412         dev_info->vmdq_pool_base     = 0;
1413         dev_info->vmdq_queue_base    = 0;
1414         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1415         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1416         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1417         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1418                                     dev_info->rx_queue_offload_capa;
1419         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1420         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1421                                     dev_info->tx_queue_offload_capa;
1422
1423         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1424         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1425
1426         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1427                 .rx_thresh = {
1428                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1429                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1430                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1431                 },
1432                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1433                 .rx_drop_en = 0,
1434                 .offloads = 0,
1435         };
1436
1437         dev_info->default_txconf = (struct rte_eth_txconf) {
1438                 .tx_thresh = {
1439                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1440                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1441                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1442                 },
1443                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1444                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1445                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1446                 .offloads = 0,
1447         };
1448
1449         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1450                 .nb_max = FM10K_MAX_RX_DESC,
1451                 .nb_min = FM10K_MIN_RX_DESC,
1452                 .nb_align = FM10K_MULT_RX_DESC,
1453         };
1454
1455         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1456                 .nb_max = FM10K_MAX_TX_DESC,
1457                 .nb_min = FM10K_MIN_TX_DESC,
1458                 .nb_align = FM10K_MULT_TX_DESC,
1459                 .nb_seg_max = FM10K_TX_MAX_SEG,
1460                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1461         };
1462
1463         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1464                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1465                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1466 }
1467
1468 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1469 static const uint32_t *
1470 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1471 {
1472         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1473             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1474                 static uint32_t ptypes[] = {
1475                         /* refers to rx_desc_to_ol_flags() */
1476                         RTE_PTYPE_L2_ETHER,
1477                         RTE_PTYPE_L3_IPV4,
1478                         RTE_PTYPE_L3_IPV4_EXT,
1479                         RTE_PTYPE_L3_IPV6,
1480                         RTE_PTYPE_L3_IPV6_EXT,
1481                         RTE_PTYPE_L4_TCP,
1482                         RTE_PTYPE_L4_UDP,
1483                         RTE_PTYPE_UNKNOWN
1484                 };
1485
1486                 return ptypes;
1487         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1488                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1489                 static uint32_t ptypes_vec[] = {
1490                         /* refers to fm10k_desc_to_pktype_v() */
1491                         RTE_PTYPE_L3_IPV4,
1492                         RTE_PTYPE_L3_IPV4_EXT,
1493                         RTE_PTYPE_L3_IPV6,
1494                         RTE_PTYPE_L3_IPV6_EXT,
1495                         RTE_PTYPE_L4_TCP,
1496                         RTE_PTYPE_L4_UDP,
1497                         RTE_PTYPE_TUNNEL_GENEVE,
1498                         RTE_PTYPE_TUNNEL_NVGRE,
1499                         RTE_PTYPE_TUNNEL_VXLAN,
1500                         RTE_PTYPE_TUNNEL_GRE,
1501                         RTE_PTYPE_UNKNOWN
1502                 };
1503
1504                 return ptypes_vec;
1505         }
1506
1507         return NULL;
1508 }
1509 #else
1510 static const uint32_t *
1511 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1512 {
1513         return NULL;
1514 }
1515 #endif
1516
1517 static int
1518 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1519 {
1520         s32 result;
1521         uint16_t mac_num = 0;
1522         uint32_t vid_idx, vid_bit, mac_index;
1523         struct fm10k_hw *hw;
1524         struct fm10k_macvlan_filter_info *macvlan;
1525         struct rte_eth_dev_data *data = dev->data;
1526
1527         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1528         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1529
1530         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1531                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1532                 return -EINVAL;
1533         }
1534
1535         if (vlan_id > ETH_VLAN_ID_MAX) {
1536                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1537                 return -EINVAL;
1538         }
1539
1540         vid_idx = FM10K_VFTA_IDX(vlan_id);
1541         vid_bit = FM10K_VFTA_BIT(vlan_id);
1542         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1543         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1544                 return 0;
1545         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1546         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1547                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1548                         "in the VLAN filter table");
1549                 return -EINVAL;
1550         }
1551
1552         fm10k_mbx_lock(hw);
1553         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1554         fm10k_mbx_unlock(hw);
1555         if (result != FM10K_SUCCESS) {
1556                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1557                 return -EIO;
1558         }
1559
1560         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1561                         (result == FM10K_SUCCESS); mac_index++) {
1562                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1563                         continue;
1564                 if (mac_num > macvlan->mac_num - 1) {
1565                         PMD_INIT_LOG(ERR, "MAC address number "
1566                                         "not match");
1567                         break;
1568                 }
1569                 fm10k_mbx_lock(hw);
1570                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1571                         data->mac_addrs[mac_index].addr_bytes,
1572                         vlan_id, on, 0);
1573                 fm10k_mbx_unlock(hw);
1574                 mac_num++;
1575         }
1576         if (result != FM10K_SUCCESS) {
1577                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1578                 return -EIO;
1579         }
1580
1581         if (on) {
1582                 macvlan->vlan_num++;
1583                 macvlan->vfta[vid_idx] |= vid_bit;
1584         } else {
1585                 macvlan->vlan_num--;
1586                 macvlan->vfta[vid_idx] &= ~vid_bit;
1587         }
1588         return 0;
1589 }
1590
1591 static int
1592 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1593 {
1594         if (mask & ETH_VLAN_STRIP_MASK) {
1595                 if (!(dev->data->dev_conf.rxmode.offloads &
1596                         DEV_RX_OFFLOAD_VLAN_STRIP))
1597                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1598                                         "always on in fm10k");
1599         }
1600
1601         if (mask & ETH_VLAN_EXTEND_MASK) {
1602                 if (dev->data->dev_conf.rxmode.offloads &
1603                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1604                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1605                                         "supported in fm10k");
1606         }
1607
1608         if (mask & ETH_VLAN_FILTER_MASK) {
1609                 if (!(dev->data->dev_conf.rxmode.offloads &
1610                         DEV_RX_OFFLOAD_VLAN_FILTER))
1611                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1612         }
1613
1614         return 0;
1615 }
1616
1617 /* Add/Remove a MAC address, and update filters to main VSI */
1618 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1619                 const u8 *mac, bool add, uint32_t pool)
1620 {
1621         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1622         struct fm10k_macvlan_filter_info *macvlan;
1623         uint32_t i, j, k;
1624
1625         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1626
1627         if (pool != MAIN_VSI_POOL_NUMBER) {
1628                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1629                         "mac to pool %u", pool);
1630                 return;
1631         }
1632         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1633                 if (!macvlan->vfta[j])
1634                         continue;
1635                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1636                         if (!(macvlan->vfta[j] & (1 << k)))
1637                                 continue;
1638                         if (i + 1 > macvlan->vlan_num) {
1639                                 PMD_INIT_LOG(ERR, "vlan number not match");
1640                                 return;
1641                         }
1642                         fm10k_mbx_lock(hw);
1643                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1644                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1645                         fm10k_mbx_unlock(hw);
1646                         i++;
1647                 }
1648         }
1649 }
1650
1651 /* Add/Remove a MAC address, and update filters to VMDQ */
1652 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1653                 const u8 *mac, bool add, uint32_t pool)
1654 {
1655         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1656         struct fm10k_macvlan_filter_info *macvlan;
1657         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1658         uint32_t i;
1659
1660         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1661         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1662
1663         if (pool > macvlan->nb_queue_pools) {
1664                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1665                         " Max pool is %u",
1666                         pool, macvlan->nb_queue_pools);
1667                 return;
1668         }
1669         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1670                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1671                         continue;
1672                 fm10k_mbx_lock(hw);
1673                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1674                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1675                 fm10k_mbx_unlock(hw);
1676         }
1677 }
1678
1679 /* Add/Remove a MAC address, and update filters */
1680 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1681                 const u8 *mac, bool add, uint32_t pool)
1682 {
1683         struct fm10k_macvlan_filter_info *macvlan;
1684
1685         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1686
1687         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1688                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1689         else
1690                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1691
1692         if (add)
1693                 macvlan->mac_num++;
1694         else
1695                 macvlan->mac_num--;
1696 }
1697
1698 /* Add a MAC address, and update filters */
1699 static int
1700 fm10k_macaddr_add(struct rte_eth_dev *dev,
1701                 struct ether_addr *mac_addr,
1702                 uint32_t index,
1703                 uint32_t pool)
1704 {
1705         struct fm10k_macvlan_filter_info *macvlan;
1706
1707         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1708         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1709         macvlan->mac_vmdq_id[index] = pool;
1710         return 0;
1711 }
1712
1713 /* Remove a MAC address, and update filters */
1714 static void
1715 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1716 {
1717         struct rte_eth_dev_data *data = dev->data;
1718         struct fm10k_macvlan_filter_info *macvlan;
1719
1720         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1721         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1722                         FALSE, macvlan->mac_vmdq_id[index]);
1723         macvlan->mac_vmdq_id[index] = 0;
1724 }
1725
1726 static inline int
1727 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1728 {
1729         if ((request < min) || (request > max) || ((request % mult) != 0))
1730                 return -1;
1731         else
1732                 return 0;
1733 }
1734
1735
1736 static inline int
1737 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1738 {
1739         if ((request < min) || (request > max) || ((div % request) != 0))
1740                 return -1;
1741         else
1742                 return 0;
1743 }
1744
1745 static inline int
1746 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1747 {
1748         uint16_t rx_free_thresh;
1749
1750         if (conf->rx_free_thresh == 0)
1751                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1752         else
1753                 rx_free_thresh = conf->rx_free_thresh;
1754
1755         /* make sure the requested threshold satisfies the constraints */
1756         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1757                         FM10K_RX_FREE_THRESH_MAX(q),
1758                         FM10K_RX_FREE_THRESH_DIV(q),
1759                         rx_free_thresh)) {
1760                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1761                         "less than or equal to %u, "
1762                         "greater than or equal to %u, "
1763                         "and a divisor of %u",
1764                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1765                         FM10K_RX_FREE_THRESH_MIN(q),
1766                         FM10K_RX_FREE_THRESH_DIV(q));
1767                 return -EINVAL;
1768         }
1769
1770         q->alloc_thresh = rx_free_thresh;
1771         q->drop_en = conf->rx_drop_en;
1772         q->rx_deferred_start = conf->rx_deferred_start;
1773
1774         return 0;
1775 }
1776
1777 /*
1778  * Hardware requires specific alignment for Rx packet buffers. At
1779  * least one of the following two conditions must be satisfied.
1780  *  1. Address is 512B aligned
1781  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1782  *
1783  * As such, the driver may need to adjust the DMA address within the
1784  * buffer by up to 512B.
1785  *
1786  * return 1 if the element size is valid, otherwise return 0.
1787  */
1788 static int
1789 mempool_element_size_valid(struct rte_mempool *mp)
1790 {
1791         uint32_t min_size;
1792
1793         /* elt_size includes mbuf header and headroom */
1794         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1795                         RTE_PKTMBUF_HEADROOM;
1796
1797         /* account for up to 512B of alignment */
1798         min_size -= FM10K_RX_DATABUF_ALIGN;
1799
1800         /* sanity check for overflow */
1801         if (min_size > mp->elt_size)
1802                 return 0;
1803
1804         /* size is valid */
1805         return 1;
1806 }
1807
1808 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1809 {
1810         RTE_SET_USED(dev);
1811
1812         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1813 }
1814
1815 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1816 {
1817         RTE_SET_USED(dev);
1818
1819         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1820                            DEV_RX_OFFLOAD_VLAN_FILTER |
1821                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1822                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1823                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1824                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1825                            DEV_RX_OFFLOAD_CRC_STRIP   |
1826                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1827 }
1828
1829 static int
1830 fm10k_check_rx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
1831 {
1832         uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
1833         uint64_t queue_supported = fm10k_get_rx_queue_offloads_capa(dev);
1834         uint64_t port_supported = fm10k_get_rx_port_offloads_capa(dev);
1835
1836         if ((requested & (queue_supported | port_supported)) != requested)
1837                 return 0;
1838
1839         if ((port_offloads ^ requested) & port_supported)
1840                 return 0;
1841
1842         return 1;
1843 }
1844
1845 static int
1846 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1847         uint16_t nb_desc, unsigned int socket_id,
1848         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1849 {
1850         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1851         struct fm10k_dev_info *dev_info =
1852                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1853         struct fm10k_rx_queue *q;
1854         const struct rte_memzone *mz;
1855
1856         PMD_INIT_FUNC_TRACE();
1857
1858         if (!fm10k_check_rx_queue_offloads(dev, conf->offloads)) {
1859                 PMD_INIT_LOG(ERR, "%p: Rx queue offloads 0x%" PRIx64
1860                         " don't match port offloads 0x%" PRIx64
1861                         " or supported port offloads 0x%" PRIx64
1862                         " or supported queue offloads 0x%" PRIx64,
1863                         (void *)dev, conf->offloads,
1864                         dev->data->dev_conf.rxmode.offloads,
1865                         fm10k_get_rx_port_offloads_capa(dev),
1866                         fm10k_get_rx_queue_offloads_capa(dev));
1867                 return -ENOTSUP;
1868         }
1869
1870         /* make sure the mempool element size can account for alignment. */
1871         if (!mempool_element_size_valid(mp)) {
1872                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1873                 return -EINVAL;
1874         }
1875
1876         /* make sure a valid number of descriptors have been requested */
1877         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1878                                 FM10K_MULT_RX_DESC, nb_desc)) {
1879                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1880                         "less than or equal to %"PRIu32", "
1881                         "greater than or equal to %u, "
1882                         "and a multiple of %u",
1883                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1884                         FM10K_MULT_RX_DESC);
1885                 return -EINVAL;
1886         }
1887
1888         /*
1889          * if this queue existed already, free the associated memory. The
1890          * queue cannot be reused in case we need to allocate memory on
1891          * different socket than was previously used.
1892          */
1893         if (dev->data->rx_queues[queue_id] != NULL) {
1894                 rx_queue_free(dev->data->rx_queues[queue_id]);
1895                 dev->data->rx_queues[queue_id] = NULL;
1896         }
1897
1898         /* allocate memory for the queue structure */
1899         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1900                                 socket_id);
1901         if (q == NULL) {
1902                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1903                 return -ENOMEM;
1904         }
1905
1906         /* setup queue */
1907         q->mp = mp;
1908         q->nb_desc = nb_desc;
1909         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1910         q->port_id = dev->data->port_id;
1911         q->queue_id = queue_id;
1912         q->tail_ptr = (volatile uint32_t *)
1913                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1914         q->offloads = conf->offloads;
1915         if (handle_rxconf(q, conf))
1916                 return -EINVAL;
1917
1918         /* allocate memory for the software ring */
1919         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1920                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1921                         RTE_CACHE_LINE_SIZE, socket_id);
1922         if (q->sw_ring == NULL) {
1923                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1924                 rte_free(q);
1925                 return -ENOMEM;
1926         }
1927
1928         /*
1929          * allocate memory for the hardware descriptor ring. A memzone large
1930          * enough to hold the maximum ring size is requested to allow for
1931          * resizing in later calls to the queue setup function.
1932          */
1933         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1934                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1935                                       socket_id);
1936         if (mz == NULL) {
1937                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1938                 rte_free(q->sw_ring);
1939                 rte_free(q);
1940                 return -ENOMEM;
1941         }
1942         q->hw_ring = mz->addr;
1943         q->hw_ring_phys_addr = mz->iova;
1944
1945         /* Check if number of descs satisfied Vector requirement */
1946         if (!rte_is_power_of_2(nb_desc)) {
1947                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1948                                     "preconditions - canceling the feature for "
1949                                     "the whole port[%d]",
1950                              q->queue_id, q->port_id);
1951                 dev_info->rx_vec_allowed = false;
1952         } else
1953                 fm10k_rxq_vec_setup(q);
1954
1955         dev->data->rx_queues[queue_id] = q;
1956         return 0;
1957 }
1958
1959 static void
1960 fm10k_rx_queue_release(void *queue)
1961 {
1962         PMD_INIT_FUNC_TRACE();
1963
1964         rx_queue_free(queue);
1965 }
1966
1967 static inline int
1968 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1969 {
1970         uint16_t tx_free_thresh;
1971         uint16_t tx_rs_thresh;
1972
1973         /* constraint MACROs require that tx_free_thresh is configured
1974          * before tx_rs_thresh */
1975         if (conf->tx_free_thresh == 0)
1976                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1977         else
1978                 tx_free_thresh = conf->tx_free_thresh;
1979
1980         /* make sure the requested threshold satisfies the constraints */
1981         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1982                         FM10K_TX_FREE_THRESH_MAX(q),
1983                         FM10K_TX_FREE_THRESH_DIV(q),
1984                         tx_free_thresh)) {
1985                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1986                         "less than or equal to %u, "
1987                         "greater than or equal to %u, "
1988                         "and a divisor of %u",
1989                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1990                         FM10K_TX_FREE_THRESH_MIN(q),
1991                         FM10K_TX_FREE_THRESH_DIV(q));
1992                 return -EINVAL;
1993         }
1994
1995         q->free_thresh = tx_free_thresh;
1996
1997         if (conf->tx_rs_thresh == 0)
1998                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1999         else
2000                 tx_rs_thresh = conf->tx_rs_thresh;
2001
2002         q->tx_deferred_start = conf->tx_deferred_start;
2003
2004         /* make sure the requested threshold satisfies the constraints */
2005         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
2006                         FM10K_TX_RS_THRESH_MAX(q),
2007                         FM10K_TX_RS_THRESH_DIV(q),
2008                         tx_rs_thresh)) {
2009                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
2010                         "less than or equal to %u, "
2011                         "greater than or equal to %u, "
2012                         "and a divisor of %u",
2013                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
2014                         FM10K_TX_RS_THRESH_MIN(q),
2015                         FM10K_TX_RS_THRESH_DIV(q));
2016                 return -EINVAL;
2017         }
2018
2019         q->rs_thresh = tx_rs_thresh;
2020
2021         return 0;
2022 }
2023
2024 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
2025 {
2026         RTE_SET_USED(dev);
2027
2028         return 0;
2029 }
2030
2031 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
2032 {
2033         RTE_SET_USED(dev);
2034
2035         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
2036                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
2037                           DEV_TX_OFFLOAD_UDP_CKSUM   |
2038                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2039                           DEV_TX_OFFLOAD_TCP_TSO);
2040 }
2041
2042 static int
2043 fm10k_check_tx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
2044 {
2045         uint64_t port_offloads = dev->data->dev_conf.txmode.offloads;
2046         uint64_t queue_supported = fm10k_get_tx_queue_offloads_capa(dev);
2047         uint64_t port_supported = fm10k_get_tx_port_offloads_capa(dev);
2048
2049         if ((requested & (queue_supported | port_supported)) != requested)
2050                 return 0;
2051
2052         if ((port_offloads ^ requested) & port_supported)
2053                 return 0;
2054
2055         return 1;
2056 }
2057
2058 static int
2059 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2060         uint16_t nb_desc, unsigned int socket_id,
2061         const struct rte_eth_txconf *conf)
2062 {
2063         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2064         struct fm10k_tx_queue *q;
2065         const struct rte_memzone *mz;
2066
2067         PMD_INIT_FUNC_TRACE();
2068
2069         if (!fm10k_check_tx_queue_offloads(dev, conf->offloads)) {
2070                 PMD_INIT_LOG(ERR, "%p: Tx queue offloads 0x%" PRIx64
2071                         " don't match port offloads 0x%" PRIx64
2072                         " or supported port offloads 0x%" PRIx64
2073                         " or supported queue offloads 0x%" PRIx64,
2074                         (void *)dev, conf->offloads,
2075                         dev->data->dev_conf.txmode.offloads,
2076                         fm10k_get_tx_port_offloads_capa(dev),
2077                         fm10k_get_tx_queue_offloads_capa(dev));
2078                 return -ENOTSUP;
2079         }
2080
2081         /* make sure a valid number of descriptors have been requested */
2082         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2083                                 FM10K_MULT_TX_DESC, nb_desc)) {
2084                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2085                         "less than or equal to %"PRIu32", "
2086                         "greater than or equal to %u, "
2087                         "and a multiple of %u",
2088                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2089                         FM10K_MULT_TX_DESC);
2090                 return -EINVAL;
2091         }
2092
2093         /*
2094          * if this queue existed already, free the associated memory. The
2095          * queue cannot be reused in case we need to allocate memory on
2096          * different socket than was previously used.
2097          */
2098         if (dev->data->tx_queues[queue_id] != NULL) {
2099                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2100
2101                 tx_queue_free(txq);
2102                 dev->data->tx_queues[queue_id] = NULL;
2103         }
2104
2105         /* allocate memory for the queue structure */
2106         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2107                                 socket_id);
2108         if (q == NULL) {
2109                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2110                 return -ENOMEM;
2111         }
2112
2113         /* setup queue */
2114         q->nb_desc = nb_desc;
2115         q->port_id = dev->data->port_id;
2116         q->queue_id = queue_id;
2117         q->txq_flags = conf->txq_flags;
2118         q->offloads = conf->offloads;
2119         q->ops = &def_txq_ops;
2120         q->tail_ptr = (volatile uint32_t *)
2121                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2122         if (handle_txconf(q, conf))
2123                 return -EINVAL;
2124
2125         /* allocate memory for the software ring */
2126         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2127                                         nb_desc * sizeof(struct rte_mbuf *),
2128                                         RTE_CACHE_LINE_SIZE, socket_id);
2129         if (q->sw_ring == NULL) {
2130                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2131                 rte_free(q);
2132                 return -ENOMEM;
2133         }
2134
2135         /*
2136          * allocate memory for the hardware descriptor ring. A memzone large
2137          * enough to hold the maximum ring size is requested to allow for
2138          * resizing in later calls to the queue setup function.
2139          */
2140         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2141                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2142                                       socket_id);
2143         if (mz == NULL) {
2144                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2145                 rte_free(q->sw_ring);
2146                 rte_free(q);
2147                 return -ENOMEM;
2148         }
2149         q->hw_ring = mz->addr;
2150         q->hw_ring_phys_addr = mz->iova;
2151
2152         /*
2153          * allocate memory for the RS bit tracker. Enough slots to hold the
2154          * descriptor index for each RS bit needing to be set are required.
2155          */
2156         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2157                                 ((nb_desc + 1) / q->rs_thresh) *
2158                                 sizeof(uint16_t),
2159                                 RTE_CACHE_LINE_SIZE, socket_id);
2160         if (q->rs_tracker.list == NULL) {
2161                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2162                 rte_free(q->sw_ring);
2163                 rte_free(q);
2164                 return -ENOMEM;
2165         }
2166
2167         dev->data->tx_queues[queue_id] = q;
2168         return 0;
2169 }
2170
2171 static void
2172 fm10k_tx_queue_release(void *queue)
2173 {
2174         struct fm10k_tx_queue *q = queue;
2175         PMD_INIT_FUNC_TRACE();
2176
2177         tx_queue_free(q);
2178 }
2179
2180 static int
2181 fm10k_reta_update(struct rte_eth_dev *dev,
2182                         struct rte_eth_rss_reta_entry64 *reta_conf,
2183                         uint16_t reta_size)
2184 {
2185         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2186         uint16_t i, j, idx, shift;
2187         uint8_t mask;
2188         uint32_t reta;
2189
2190         PMD_INIT_FUNC_TRACE();
2191
2192         if (reta_size > FM10K_MAX_RSS_INDICES) {
2193                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2194                         "(%d) doesn't match the number hardware can supported "
2195                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2196                 return -EINVAL;
2197         }
2198
2199         /*
2200          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2201          * 128-entries in 32 registers
2202          */
2203         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2204                 idx = i / RTE_RETA_GROUP_SIZE;
2205                 shift = i % RTE_RETA_GROUP_SIZE;
2206                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2207                                 BIT_MASK_PER_UINT32);
2208                 if (mask == 0)
2209                         continue;
2210
2211                 reta = 0;
2212                 if (mask != BIT_MASK_PER_UINT32)
2213                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2214
2215                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2216                         if (mask & (0x1 << j)) {
2217                                 if (mask != 0xF)
2218                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2219                                 reta |= reta_conf[idx].reta[shift + j] <<
2220                                                 (CHAR_BIT * j);
2221                         }
2222                 }
2223                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2224         }
2225
2226         return 0;
2227 }
2228
2229 static int
2230 fm10k_reta_query(struct rte_eth_dev *dev,
2231                         struct rte_eth_rss_reta_entry64 *reta_conf,
2232                         uint16_t reta_size)
2233 {
2234         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2235         uint16_t i, j, idx, shift;
2236         uint8_t mask;
2237         uint32_t reta;
2238
2239         PMD_INIT_FUNC_TRACE();
2240
2241         if (reta_size < FM10K_MAX_RSS_INDICES) {
2242                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2243                         "(%d) doesn't match the number hardware can supported "
2244                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2245                 return -EINVAL;
2246         }
2247
2248         /*
2249          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2250          * 128-entries in 32 registers
2251          */
2252         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2253                 idx = i / RTE_RETA_GROUP_SIZE;
2254                 shift = i % RTE_RETA_GROUP_SIZE;
2255                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2256                                 BIT_MASK_PER_UINT32);
2257                 if (mask == 0)
2258                         continue;
2259
2260                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2261                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2262                         if (mask & (0x1 << j))
2263                                 reta_conf[idx].reta[shift + j] = ((reta >>
2264                                         CHAR_BIT * j) & UINT8_MAX);
2265                 }
2266         }
2267
2268         return 0;
2269 }
2270
2271 static int
2272 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2273         struct rte_eth_rss_conf *rss_conf)
2274 {
2275         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2276         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2277         uint32_t mrqc;
2278         uint64_t hf = rss_conf->rss_hf;
2279         int i;
2280
2281         PMD_INIT_FUNC_TRACE();
2282
2283         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2284                                 FM10K_RSSRK_ENTRIES_PER_REG))
2285                 return -EINVAL;
2286
2287         if (hf == 0)
2288                 return -EINVAL;
2289
2290         mrqc = 0;
2291         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2292         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2293         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2294         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2295         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2296         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2297         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2298         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2299         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2300
2301         /* If the mapping doesn't fit any supported, return */
2302         if (mrqc == 0)
2303                 return -EINVAL;
2304
2305         if (key != NULL)
2306                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2307                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2308
2309         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2310
2311         return 0;
2312 }
2313
2314 static int
2315 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2316         struct rte_eth_rss_conf *rss_conf)
2317 {
2318         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2319         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2320         uint32_t mrqc;
2321         uint64_t hf;
2322         int i;
2323
2324         PMD_INIT_FUNC_TRACE();
2325
2326         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2327                                 FM10K_RSSRK_ENTRIES_PER_REG))
2328                 return -EINVAL;
2329
2330         if (key != NULL)
2331                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2332                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2333
2334         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2335         hf = 0;
2336         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2337         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2338         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2339         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2340         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2341         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2342         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2343         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2344         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2345
2346         rss_conf->rss_hf = hf;
2347
2348         return 0;
2349 }
2350
2351 static void
2352 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2353 {
2354         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2355         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2356
2357         /* Bind all local non-queue interrupt to vector 0 */
2358         int_map |= FM10K_MISC_VEC_ID;
2359
2360         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2361         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2362         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2363         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2364         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2365         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2366
2367         /* Enable misc causes */
2368         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2369                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2370                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2371                                 FM10K_EIMR_ENABLE(MAILBOX) |
2372                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2373                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2374                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2375                                 FM10K_EIMR_ENABLE(VFLR));
2376
2377         /* Enable ITR 0 */
2378         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2379                                         FM10K_ITR_MASK_CLEAR);
2380         FM10K_WRITE_FLUSH(hw);
2381 }
2382
2383 static void
2384 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2385 {
2386         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2387         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2388
2389         int_map |= FM10K_MISC_VEC_ID;
2390
2391         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2392         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2393         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2394         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2395         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2396         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2397
2398         /* Disable misc causes */
2399         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2400                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2401                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2402                                 FM10K_EIMR_DISABLE(MAILBOX) |
2403                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2404                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2405                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2406                                 FM10K_EIMR_DISABLE(VFLR));
2407
2408         /* Disable ITR 0 */
2409         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2410         FM10K_WRITE_FLUSH(hw);
2411 }
2412
2413 static void
2414 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2415 {
2416         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2417         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2418
2419         /* Bind all local non-queue interrupt to vector 0 */
2420         int_map |= FM10K_MISC_VEC_ID;
2421
2422         /* Only INT 0 available, other 15 are reserved. */
2423         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2424
2425         /* Enable ITR 0 */
2426         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2427                                         FM10K_ITR_MASK_CLEAR);
2428         FM10K_WRITE_FLUSH(hw);
2429 }
2430
2431 static void
2432 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2433 {
2434         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2435         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2436
2437         int_map |= FM10K_MISC_VEC_ID;
2438
2439         /* Only INT 0 available, other 15 are reserved. */
2440         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2441
2442         /* Disable ITR 0 */
2443         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2444         FM10K_WRITE_FLUSH(hw);
2445 }
2446
2447 static int
2448 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2449 {
2450         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2451         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2452
2453         /* Enable ITR */
2454         if (hw->mac.type == fm10k_mac_pf)
2455                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2456                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2457         else
2458                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2459                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2460         rte_intr_enable(&pdev->intr_handle);
2461         return 0;
2462 }
2463
2464 static int
2465 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2466 {
2467         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2468         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2469
2470         /* Disable ITR */
2471         if (hw->mac.type == fm10k_mac_pf)
2472                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2473                         FM10K_ITR_MASK_SET);
2474         else
2475                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2476                         FM10K_ITR_MASK_SET);
2477         return 0;
2478 }
2479
2480 static int
2481 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2482 {
2483         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2484         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2485         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2486         uint32_t intr_vector, vec;
2487         uint16_t queue_id;
2488         int result = 0;
2489
2490         /* fm10k needs one separate interrupt for mailbox,
2491          * so only drivers which support multiple interrupt vectors
2492          * e.g. vfio-pci can work for fm10k interrupt mode
2493          */
2494         if (!rte_intr_cap_multiple(intr_handle) ||
2495                         dev->data->dev_conf.intr_conf.rxq == 0)
2496                 return result;
2497
2498         intr_vector = dev->data->nb_rx_queues;
2499
2500         /* disable interrupt first */
2501         rte_intr_disable(intr_handle);
2502         if (hw->mac.type == fm10k_mac_pf)
2503                 fm10k_dev_disable_intr_pf(dev);
2504         else
2505                 fm10k_dev_disable_intr_vf(dev);
2506
2507         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2508                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2509                 result = -EIO;
2510         }
2511
2512         if (rte_intr_dp_is_en(intr_handle) && !result) {
2513                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2514                         dev->data->nb_rx_queues * sizeof(int), 0);
2515                 if (intr_handle->intr_vec) {
2516                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2517                                         queue_id < dev->data->nb_rx_queues;
2518                                         queue_id++) {
2519                                 intr_handle->intr_vec[queue_id] = vec;
2520                                 if (vec < intr_handle->nb_efd - 1
2521                                                 + FM10K_RX_VEC_START)
2522                                         vec++;
2523                         }
2524                 } else {
2525                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2526                                 " intr_vec", dev->data->nb_rx_queues);
2527                         rte_intr_efd_disable(intr_handle);
2528                         result = -ENOMEM;
2529                 }
2530         }
2531
2532         if (hw->mac.type == fm10k_mac_pf)
2533                 fm10k_dev_enable_intr_pf(dev);
2534         else
2535                 fm10k_dev_enable_intr_vf(dev);
2536         rte_intr_enable(intr_handle);
2537         hw->mac.ops.update_int_moderator(hw);
2538         return result;
2539 }
2540
2541 static int
2542 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2543 {
2544         struct fm10k_fault fault;
2545         int err;
2546         const char *estr = "Unknown error";
2547
2548         /* Process PCA fault */
2549         if (eicr & FM10K_EICR_PCA_FAULT) {
2550                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2551                 if (err)
2552                         goto error;
2553                 switch (fault.type) {
2554                 case PCA_NO_FAULT:
2555                         estr = "PCA_NO_FAULT"; break;
2556                 case PCA_UNMAPPED_ADDR:
2557                         estr = "PCA_UNMAPPED_ADDR"; break;
2558                 case PCA_BAD_QACCESS_PF:
2559                         estr = "PCA_BAD_QACCESS_PF"; break;
2560                 case PCA_BAD_QACCESS_VF:
2561                         estr = "PCA_BAD_QACCESS_VF"; break;
2562                 case PCA_MALICIOUS_REQ:
2563                         estr = "PCA_MALICIOUS_REQ"; break;
2564                 case PCA_POISONED_TLP:
2565                         estr = "PCA_POISONED_TLP"; break;
2566                 case PCA_TLP_ABORT:
2567                         estr = "PCA_TLP_ABORT"; break;
2568                 default:
2569                         goto error;
2570                 }
2571                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2572                         estr, fault.func ? "VF" : "PF", fault.func,
2573                         fault.address, fault.specinfo);
2574         }
2575
2576         /* Process THI fault */
2577         if (eicr & FM10K_EICR_THI_FAULT) {
2578                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2579                 if (err)
2580                         goto error;
2581                 switch (fault.type) {
2582                 case THI_NO_FAULT:
2583                         estr = "THI_NO_FAULT"; break;
2584                 case THI_MAL_DIS_Q_FAULT:
2585                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2586                 default:
2587                         goto error;
2588                 }
2589                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2590                         estr, fault.func ? "VF" : "PF", fault.func,
2591                         fault.address, fault.specinfo);
2592         }
2593
2594         /* Process FUM fault */
2595         if (eicr & FM10K_EICR_FUM_FAULT) {
2596                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2597                 if (err)
2598                         goto error;
2599                 switch (fault.type) {
2600                 case FUM_NO_FAULT:
2601                         estr = "FUM_NO_FAULT"; break;
2602                 case FUM_UNMAPPED_ADDR:
2603                         estr = "FUM_UNMAPPED_ADDR"; break;
2604                 case FUM_POISONED_TLP:
2605                         estr = "FUM_POISONED_TLP"; break;
2606                 case FUM_BAD_VF_QACCESS:
2607                         estr = "FUM_BAD_VF_QACCESS"; break;
2608                 case FUM_ADD_DECODE_ERR:
2609                         estr = "FUM_ADD_DECODE_ERR"; break;
2610                 case FUM_RO_ERROR:
2611                         estr = "FUM_RO_ERROR"; break;
2612                 case FUM_QPRC_CRC_ERROR:
2613                         estr = "FUM_QPRC_CRC_ERROR"; break;
2614                 case FUM_CSR_TIMEOUT:
2615                         estr = "FUM_CSR_TIMEOUT"; break;
2616                 case FUM_INVALID_TYPE:
2617                         estr = "FUM_INVALID_TYPE"; break;
2618                 case FUM_INVALID_LENGTH:
2619                         estr = "FUM_INVALID_LENGTH"; break;
2620                 case FUM_INVALID_BE:
2621                         estr = "FUM_INVALID_BE"; break;
2622                 case FUM_INVALID_ALIGN:
2623                         estr = "FUM_INVALID_ALIGN"; break;
2624                 default:
2625                         goto error;
2626                 }
2627                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2628                         estr, fault.func ? "VF" : "PF", fault.func,
2629                         fault.address, fault.specinfo);
2630         }
2631
2632         return 0;
2633 error:
2634         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2635         return err;
2636 }
2637
2638 /**
2639  * PF interrupt handler triggered by NIC for handling specific interrupt.
2640  *
2641  * @param handle
2642  *  Pointer to interrupt handle.
2643  * @param param
2644  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2645  *
2646  * @return
2647  *  void
2648  */
2649 static void
2650 fm10k_dev_interrupt_handler_pf(void *param)
2651 {
2652         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2653         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2654         uint32_t cause, status;
2655         struct fm10k_dev_info *dev_info =
2656                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2657         int status_mbx;
2658         s32 err;
2659
2660         if (hw->mac.type != fm10k_mac_pf)
2661                 return;
2662
2663         cause = FM10K_READ_REG(hw, FM10K_EICR);
2664
2665         /* Handle PCI fault cases */
2666         if (cause & FM10K_EICR_FAULT_MASK) {
2667                 PMD_INIT_LOG(ERR, "INT: find fault!");
2668                 fm10k_dev_handle_fault(hw, cause);
2669         }
2670
2671         /* Handle switch up/down */
2672         if (cause & FM10K_EICR_SWITCHNOTREADY)
2673                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2674
2675         if (cause & FM10K_EICR_SWITCHREADY) {
2676                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2677                 if (dev_info->sm_down == 1) {
2678                         fm10k_mbx_lock(hw);
2679
2680                         /* For recreating logical ports */
2681                         status_mbx = hw->mac.ops.update_lport_state(hw,
2682                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2683                         if (status_mbx == FM10K_SUCCESS)
2684                                 PMD_INIT_LOG(INFO,
2685                                         "INT: Recreated Logical port");
2686                         else
2687                                 PMD_INIT_LOG(INFO,
2688                                         "INT: Logical ports weren't recreated");
2689
2690                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2691                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2692                         if (status_mbx != FM10K_SUCCESS)
2693                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2694
2695                         fm10k_mbx_unlock(hw);
2696
2697                         /* first clear the internal SW recording structure */
2698                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2699                                                 ETH_MQ_RX_VMDQ_FLAG))
2700                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2701                                         false);
2702
2703                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2704                                         MAIN_VSI_POOL_NUMBER);
2705
2706                         /*
2707                          * Add default mac address and vlan for the logical
2708                          * ports that have been created, leave to the
2709                          * application to fully recover Rx filtering.
2710                          */
2711                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2712                                         MAIN_VSI_POOL_NUMBER);
2713
2714                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2715                                                 ETH_MQ_RX_VMDQ_FLAG))
2716                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2717                                         true);
2718
2719                         dev_info->sm_down = 0;
2720                         _rte_eth_dev_callback_process(dev,
2721                                         RTE_ETH_EVENT_INTR_LSC,
2722                                         NULL);
2723                 }
2724         }
2725
2726         /* Handle mailbox message */
2727         fm10k_mbx_lock(hw);
2728         err = hw->mbx.ops.process(hw, &hw->mbx);
2729         fm10k_mbx_unlock(hw);
2730
2731         if (err == FM10K_ERR_RESET_REQUESTED) {
2732                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2733                 dev_info->sm_down = 1;
2734                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2735                                 NULL);
2736         }
2737
2738         /* Handle SRAM error */
2739         if (cause & FM10K_EICR_SRAMERROR) {
2740                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2741
2742                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2743                 /* Write to clear pending bits */
2744                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2745
2746                 /* Todo: print out error message after shared code  updates */
2747         }
2748
2749         /* Clear these 3 events if having any */
2750         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2751                  FM10K_EICR_SWITCHREADY;
2752         if (cause)
2753                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2754
2755         /* Re-enable interrupt from device side */
2756         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2757                                         FM10K_ITR_MASK_CLEAR);
2758         /* Re-enable interrupt from host side */
2759         rte_intr_enable(dev->intr_handle);
2760 }
2761
2762 /**
2763  * VF interrupt handler triggered by NIC for handling specific interrupt.
2764  *
2765  * @param handle
2766  *  Pointer to interrupt handle.
2767  * @param param
2768  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2769  *
2770  * @return
2771  *  void
2772  */
2773 static void
2774 fm10k_dev_interrupt_handler_vf(void *param)
2775 {
2776         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2777         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2778         struct fm10k_mbx_info *mbx = &hw->mbx;
2779         struct fm10k_dev_info *dev_info =
2780                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2781         const enum fm10k_mbx_state state = mbx->state;
2782         int status_mbx;
2783
2784         if (hw->mac.type != fm10k_mac_vf)
2785                 return;
2786
2787         /* Handle mailbox message if lock is acquired */
2788         fm10k_mbx_lock(hw);
2789         hw->mbx.ops.process(hw, &hw->mbx);
2790         fm10k_mbx_unlock(hw);
2791
2792         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2793                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2794
2795                 fm10k_mbx_lock(hw);
2796                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2797                                 MAX_LPORT_NUM, 1);
2798                 fm10k_mbx_unlock(hw);
2799
2800                 /* Setting reset flag */
2801                 dev_info->sm_down = 1;
2802                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2803                                 NULL);
2804         }
2805
2806         if (dev_info->sm_down == 1 &&
2807                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2808                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2809                 fm10k_mbx_lock(hw);
2810                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2811                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2812                 if (status_mbx != FM10K_SUCCESS)
2813                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2814                 fm10k_mbx_unlock(hw);
2815
2816                 /* first clear the internal SW recording structure */
2817                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2818                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2819                                 MAIN_VSI_POOL_NUMBER);
2820
2821                 /*
2822                  * Add default mac address and vlan for the logical ports that
2823                  * have been created, leave to the application to fully recover
2824                  * Rx filtering.
2825                  */
2826                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2827                                 MAIN_VSI_POOL_NUMBER);
2828                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2829
2830                 dev_info->sm_down = 0;
2831                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2832                                 NULL);
2833         }
2834
2835         /* Re-enable interrupt from device side */
2836         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2837                                         FM10K_ITR_MASK_CLEAR);
2838         /* Re-enable interrupt from host side */
2839         rte_intr_enable(dev->intr_handle);
2840 }
2841
2842 /* Mailbox message handler in VF */
2843 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2844         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2845         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2846         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2847         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2848 };
2849
2850 static int
2851 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2852 {
2853         int err = 0;
2854
2855         /* Initialize mailbox lock */
2856         fm10k_mbx_initlock(hw);
2857
2858         /* Replace default message handler with new ones */
2859         if (hw->mac.type == fm10k_mac_vf)
2860                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2861
2862         if (err) {
2863                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2864                                 err);
2865                 return err;
2866         }
2867         /* Connect to SM for PF device or PF for VF device */
2868         return hw->mbx.ops.connect(hw, &hw->mbx);
2869 }
2870
2871 static void
2872 fm10k_close_mbx_service(struct fm10k_hw *hw)
2873 {
2874         /* Disconnect from SM for PF device or PF for VF device */
2875         hw->mbx.ops.disconnect(hw, &hw->mbx);
2876 }
2877
2878 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2879         .dev_configure          = fm10k_dev_configure,
2880         .dev_start              = fm10k_dev_start,
2881         .dev_stop               = fm10k_dev_stop,
2882         .dev_close              = fm10k_dev_close,
2883         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2884         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2885         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2886         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2887         .stats_get              = fm10k_stats_get,
2888         .xstats_get             = fm10k_xstats_get,
2889         .xstats_get_names       = fm10k_xstats_get_names,
2890         .stats_reset            = fm10k_stats_reset,
2891         .xstats_reset           = fm10k_stats_reset,
2892         .link_update            = fm10k_link_update,
2893         .dev_infos_get          = fm10k_dev_infos_get,
2894         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2895         .vlan_filter_set        = fm10k_vlan_filter_set,
2896         .vlan_offload_set       = fm10k_vlan_offload_set,
2897         .mac_addr_add           = fm10k_macaddr_add,
2898         .mac_addr_remove        = fm10k_macaddr_remove,
2899         .rx_queue_start         = fm10k_dev_rx_queue_start,
2900         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2901         .tx_queue_start         = fm10k_dev_tx_queue_start,
2902         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2903         .rx_queue_setup         = fm10k_rx_queue_setup,
2904         .rx_queue_release       = fm10k_rx_queue_release,
2905         .tx_queue_setup         = fm10k_tx_queue_setup,
2906         .tx_queue_release       = fm10k_tx_queue_release,
2907         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2908         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2909         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2910         .reta_update            = fm10k_reta_update,
2911         .reta_query             = fm10k_reta_query,
2912         .rss_hash_update        = fm10k_rss_hash_update,
2913         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2914 };
2915
2916 static int ftag_check_handler(__rte_unused const char *key,
2917                 const char *value, __rte_unused void *opaque)
2918 {
2919         if (strcmp(value, "1"))
2920                 return -1;
2921
2922         return 0;
2923 }
2924
2925 static int
2926 fm10k_check_ftag(struct rte_devargs *devargs)
2927 {
2928         struct rte_kvargs *kvlist;
2929         const char *ftag_key = "enable_ftag";
2930
2931         if (devargs == NULL)
2932                 return 0;
2933
2934         kvlist = rte_kvargs_parse(devargs->args, NULL);
2935         if (kvlist == NULL)
2936                 return 0;
2937
2938         if (!rte_kvargs_count(kvlist, ftag_key)) {
2939                 rte_kvargs_free(kvlist);
2940                 return 0;
2941         }
2942         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2943         if (rte_kvargs_process(kvlist, ftag_key,
2944                                 ftag_check_handler, NULL) < 0) {
2945                 rte_kvargs_free(kvlist);
2946                 return 0;
2947         }
2948         rte_kvargs_free(kvlist);
2949
2950         return 1;
2951 }
2952
2953 static uint16_t
2954 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2955                     uint16_t nb_pkts)
2956 {
2957         uint16_t nb_tx = 0;
2958         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2959
2960         while (nb_pkts) {
2961                 uint16_t ret, num;
2962
2963                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2964                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2965                                                  num);
2966                 nb_tx += ret;
2967                 nb_pkts -= ret;
2968                 if (ret < num)
2969                         break;
2970         }
2971
2972         return nb_tx;
2973 }
2974
2975 static void __attribute__((cold))
2976 fm10k_set_tx_function(struct rte_eth_dev *dev)
2977 {
2978         struct fm10k_tx_queue *txq;
2979         int i;
2980         int use_sse = 1;
2981         uint16_t tx_ftag_en = 0;
2982
2983         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2984                 /* primary process has set the ftag flag and txq_flags */
2985                 txq = dev->data->tx_queues[0];
2986                 if (fm10k_tx_vec_condition_check(txq)) {
2987                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2988                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2989                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2990                 } else {
2991                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2992                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2993                         dev->tx_pkt_prepare = NULL;
2994                 }
2995                 return;
2996         }
2997
2998         if (fm10k_check_ftag(dev->device->devargs))
2999                 tx_ftag_en = 1;
3000
3001         for (i = 0; i < dev->data->nb_tx_queues; i++) {
3002                 txq = dev->data->tx_queues[i];
3003                 txq->tx_ftag_en = tx_ftag_en;
3004                 /* Check if Vector Tx is satisfied */
3005                 if (fm10k_tx_vec_condition_check(txq))
3006                         use_sse = 0;
3007         }
3008
3009         if (use_sse) {
3010                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
3011                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3012                         txq = dev->data->tx_queues[i];
3013                         fm10k_txq_vec_setup(txq);
3014                 }
3015                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
3016                 dev->tx_pkt_prepare = NULL;
3017         } else {
3018                 dev->tx_pkt_burst = fm10k_xmit_pkts;
3019                 dev->tx_pkt_prepare = fm10k_prep_pkts;
3020                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
3021         }
3022 }
3023
3024 static void __attribute__((cold))
3025 fm10k_set_rx_function(struct rte_eth_dev *dev)
3026 {
3027         struct fm10k_dev_info *dev_info =
3028                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3029         uint16_t i, rx_using_sse;
3030         uint16_t rx_ftag_en = 0;
3031
3032         if (fm10k_check_ftag(dev->device->devargs))
3033                 rx_ftag_en = 1;
3034
3035         /* In order to allow Vector Rx there are a few configuration
3036          * conditions to be met.
3037          */
3038         if (!fm10k_rx_vec_condition_check(dev) &&
3039                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3040                 if (dev->data->scattered_rx)
3041                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3042                 else
3043                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3044         } else if (dev->data->scattered_rx)
3045                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3046         else
3047                 dev->rx_pkt_burst = fm10k_recv_pkts;
3048
3049         rx_using_sse =
3050                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3051                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3052
3053         if (rx_using_sse)
3054                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3055         else
3056                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3057
3058         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3059                 return;
3060
3061         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3062                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3063
3064                 rxq->rx_using_sse = rx_using_sse;
3065                 rxq->rx_ftag_en = rx_ftag_en;
3066         }
3067 }
3068
3069 static void
3070 fm10k_params_init(struct rte_eth_dev *dev)
3071 {
3072         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3073         struct fm10k_dev_info *info =
3074                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3075
3076         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3077          * there is no way to get link status without reading BAR4.  Until this
3078          * works, assume we have maximum bandwidth.
3079          * @todo - fix bus info
3080          */
3081         hw->bus_caps.speed = fm10k_bus_speed_8000;
3082         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3083         hw->bus_caps.payload = fm10k_bus_payload_512;
3084         hw->bus.speed = fm10k_bus_speed_8000;
3085         hw->bus.width = fm10k_bus_width_pcie_x8;
3086         hw->bus.payload = fm10k_bus_payload_256;
3087
3088         info->rx_vec_allowed = true;
3089 }
3090
3091 static int
3092 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3093 {
3094         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3095         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3096         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3097         int diag, i;
3098         struct fm10k_macvlan_filter_info *macvlan;
3099
3100         PMD_INIT_FUNC_TRACE();
3101
3102         dev->dev_ops = &fm10k_eth_dev_ops;
3103         dev->rx_pkt_burst = &fm10k_recv_pkts;
3104         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3105         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3106
3107         /*
3108          * Primary process does the whole initialization, for secondary
3109          * processes, we just select the same Rx and Tx function as primary.
3110          */
3111         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3112                 fm10k_set_rx_function(dev);
3113                 fm10k_set_tx_function(dev);
3114                 return 0;
3115         }
3116
3117         rte_eth_copy_pci_info(dev, pdev);
3118
3119         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3120         memset(macvlan, 0, sizeof(*macvlan));
3121         /* Vendor and Device ID need to be set before init of shared code */
3122         memset(hw, 0, sizeof(*hw));
3123         hw->device_id = pdev->id.device_id;
3124         hw->vendor_id = pdev->id.vendor_id;
3125         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3126         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3127         hw->revision_id = 0;
3128         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3129         if (hw->hw_addr == NULL) {
3130                 PMD_INIT_LOG(ERR, "Bad mem resource."
3131                         " Try to blacklist unused devices.");
3132                 return -EIO;
3133         }
3134
3135         /* Store fm10k_adapter pointer */
3136         hw->back = dev->data->dev_private;
3137
3138         /* Initialize the shared code */
3139         diag = fm10k_init_shared_code(hw);
3140         if (diag != FM10K_SUCCESS) {
3141                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3142                 return -EIO;
3143         }
3144
3145         /* Initialize parameters */
3146         fm10k_params_init(dev);
3147
3148         /* Initialize the hw */
3149         diag = fm10k_init_hw(hw);
3150         if (diag != FM10K_SUCCESS) {
3151                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3152                 return -EIO;
3153         }
3154
3155         /* Initialize MAC address(es) */
3156         dev->data->mac_addrs = rte_zmalloc("fm10k",
3157                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3158         if (dev->data->mac_addrs == NULL) {
3159                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3160                 return -ENOMEM;
3161         }
3162
3163         diag = fm10k_read_mac_addr(hw);
3164
3165         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3166                         &dev->data->mac_addrs[0]);
3167
3168         if (diag != FM10K_SUCCESS ||
3169                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3170
3171                 /* Generate a random addr */
3172                 eth_random_addr(hw->mac.addr);
3173                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3174                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3175                 &dev->data->mac_addrs[0]);
3176         }
3177
3178         /* Reset the hw statistics */
3179         fm10k_stats_reset(dev);
3180
3181         /* Reset the hw */
3182         diag = fm10k_reset_hw(hw);
3183         if (diag != FM10K_SUCCESS) {
3184                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3185                 return -EIO;
3186         }
3187
3188         /* Setup mailbox service */
3189         diag = fm10k_setup_mbx_service(hw);
3190         if (diag != FM10K_SUCCESS) {
3191                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3192                 return -EIO;
3193         }
3194
3195         /*PF/VF has different interrupt handling mechanism */
3196         if (hw->mac.type == fm10k_mac_pf) {
3197                 /* register callback func to eal lib */
3198                 rte_intr_callback_register(intr_handle,
3199                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3200
3201                 /* enable MISC interrupt */
3202                 fm10k_dev_enable_intr_pf(dev);
3203         } else { /* VF */
3204                 rte_intr_callback_register(intr_handle,
3205                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3206
3207                 fm10k_dev_enable_intr_vf(dev);
3208         }
3209
3210         /* Enable intr after callback registered */
3211         rte_intr_enable(intr_handle);
3212
3213         hw->mac.ops.update_int_moderator(hw);
3214
3215         /* Make sure Switch Manager is ready before going forward. */
3216         if (hw->mac.type == fm10k_mac_pf) {
3217                 int switch_ready = 0;
3218
3219                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3220                         fm10k_mbx_lock(hw);
3221                         hw->mac.ops.get_host_state(hw, &switch_ready);
3222                         fm10k_mbx_unlock(hw);
3223                         if (switch_ready)
3224                                 break;
3225                         /* Delay some time to acquire async LPORT_MAP info. */
3226                         rte_delay_us(WAIT_SWITCH_MSG_US);
3227                 }
3228
3229                 if (switch_ready == 0) {
3230                         PMD_INIT_LOG(ERR, "switch is not ready");
3231                         return -1;
3232                 }
3233         }
3234
3235         /*
3236          * Below function will trigger operations on mailbox, acquire lock to
3237          * avoid race condition from interrupt handler. Operations on mailbox
3238          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3239          * will handle and generate an interrupt to our side. Then,  FIFO in
3240          * mailbox will be touched.
3241          */
3242         fm10k_mbx_lock(hw);
3243         /* Enable port first */
3244         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3245                                         MAX_LPORT_NUM, 1);
3246
3247         /* Set unicast mode by default. App can change to other mode in other
3248          * API func.
3249          */
3250         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3251                                         FM10K_XCAST_MODE_NONE);
3252
3253         fm10k_mbx_unlock(hw);
3254
3255         /* Make sure default VID is ready before going forward. */
3256         if (hw->mac.type == fm10k_mac_pf) {
3257                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3258                         if (hw->mac.default_vid)
3259                                 break;
3260                         /* Delay some time to acquire async port VLAN info. */
3261                         rte_delay_us(WAIT_SWITCH_MSG_US);
3262                 }
3263
3264                 if (!hw->mac.default_vid) {
3265                         PMD_INIT_LOG(ERR, "default VID is not ready");
3266                         return -1;
3267                 }
3268         }
3269
3270         /* Add default mac address */
3271         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3272                 MAIN_VSI_POOL_NUMBER);
3273
3274         return 0;
3275 }
3276
3277 static int
3278 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3279 {
3280         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3281         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3282         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3283         PMD_INIT_FUNC_TRACE();
3284
3285         /* only uninitialize in the primary process */
3286         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3287                 return 0;
3288
3289         /* safe to close dev here */
3290         fm10k_dev_close(dev);
3291
3292         dev->dev_ops = NULL;
3293         dev->rx_pkt_burst = NULL;
3294         dev->tx_pkt_burst = NULL;
3295
3296         /* disable uio/vfio intr */
3297         rte_intr_disable(intr_handle);
3298
3299         /*PF/VF has different interrupt handling mechanism */
3300         if (hw->mac.type == fm10k_mac_pf) {
3301                 /* disable interrupt */
3302                 fm10k_dev_disable_intr_pf(dev);
3303
3304                 /* unregister callback func to eal lib */
3305                 rte_intr_callback_unregister(intr_handle,
3306                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3307         } else {
3308                 /* disable interrupt */
3309                 fm10k_dev_disable_intr_vf(dev);
3310
3311                 rte_intr_callback_unregister(intr_handle,
3312                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3313         }
3314
3315         /* free mac memory */
3316         if (dev->data->mac_addrs) {
3317                 rte_free(dev->data->mac_addrs);
3318                 dev->data->mac_addrs = NULL;
3319         }
3320
3321         memset(hw, 0, sizeof(*hw));
3322
3323         return 0;
3324 }
3325
3326 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3327         struct rte_pci_device *pci_dev)
3328 {
3329         return rte_eth_dev_pci_generic_probe(pci_dev,
3330                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3331 }
3332
3333 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3334 {
3335         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3336 }
3337
3338 /*
3339  * The set of PCI devices this driver supports. This driver will enable both PF
3340  * and SRIOV-VF devices.
3341  */
3342 static const struct rte_pci_id pci_id_fm10k_map[] = {
3343         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3344         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3345         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3346         { .vendor_id = 0, /* sentinel */ },
3347 };
3348
3349 static struct rte_pci_driver rte_pmd_fm10k = {
3350         .id_table = pci_id_fm10k_map,
3351         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3352                      RTE_PCI_DRV_IOVA_AS_VA,
3353         .probe = eth_fm10k_pci_probe,
3354         .remove = eth_fm10k_pci_remove,
3355 };
3356
3357 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3358 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3359 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3360
3361 RTE_INIT(fm10k_init_log);
3362 static void
3363 fm10k_init_log(void)
3364 {
3365         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3366         if (fm10k_logtype_init >= 0)
3367                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3368         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3369         if (fm10k_logtype_driver >= 0)
3370                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3371 }