drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451         struct rte_eth_dev_info dev_info;
 452         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
 453         uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
 454
 455         PMD_INIT_FUNC_TRACE();
 456
 457         if ((rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP) == 0)
 458                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 459
 460         fm10k_dev_infos_get(dev, &dev_info);
 461         if ((rx_offloads & dev_info.rx_offload_capa) != rx_offloads) {
 462                 PMD_DRV_LOG(ERR, "Some Rx offloads are not supported "
 463                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 464                             rx_offloads, dev_info.rx_offload_capa);
 465                 return -ENOTSUP;
 466         }
 467         if ((tx_offloads & dev_info.tx_offload_capa) != tx_offloads) {
 468                 PMD_DRV_LOG(ERR, "Some Tx offloads are not supported "
 469                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 470                             tx_offloads, dev_info.tx_offload_capa);
 471                 return -ENOTSUP;
 472         }
 473
 474         /* multipe queue mode checking */
 475         ret  = fm10k_check_mq_mode(dev);
 476         if (ret != 0) {
 477                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 478                             ret);
 479                 return ret;
 480         }
 481
 482         dev->data->scattered_rx = 0;
 483
 484         return 0;
 485 }
 486
 487 /* fls = find last set bit = 32 minus the number of leading zeros */
 488 #ifndef fls
 489 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 490 #endif
 491
 492 static void
 493 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 494 {
 495         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 496         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 497         uint32_t i;
 498
 499         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 500
 501         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 502                 if (!vmdq_conf->pool_map[i].pools)
 503                         continue;
 504                 fm10k_mbx_lock(hw);
 505                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 506                 fm10k_mbx_unlock(hw);
 507         }
 508 }
 509
 510 static void
 511 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 512 {
 513         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 514
 515         /* Add default mac address */
 516         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 517                 MAIN_VSI_POOL_NUMBER);
 518 }
 519
 520 static void
 521 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 522 {
 523         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 524         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 525         uint32_t mrqc, *key, i, reta, j;
 526         uint64_t hf;
 527
 528 #define RSS_KEY_SIZE 40
 529         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 530                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 531                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 532                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 533                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 534                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 535         };
 536
 537         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 538                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 539                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 540                 return;
 541         }
 542
 543         /* random key is rss_intel_key (default) or user provided (rss_key) */
 544         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 545                 key = (uint32_t *)rss_intel_key;
 546         else
 547                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 548
 549         /* Now fill our hash function seeds, 4 bytes at a time */
 550         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 551                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 552
 553         /*
 554          * Fill in redirection table
 555          * The byte-swap is needed because NIC registers are in
 556          * little-endian order.
 557          */
 558         reta = 0;
 559         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 560                 if (j == dev->data->nb_rx_queues)
 561                         j = 0;
 562                 reta = (reta << CHAR_BIT) | j;
 563                 if ((i & 3) == 3)
 564                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 565                                         rte_bswap32(reta));
 566         }
 567
 568         /*
 569          * Generate RSS hash based on packet types, TCP/UDP
 570          * port numbers and/or IPv4/v6 src and dst addresses
 571          */
 572         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 573         mrqc = 0;
 574         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 578         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 581         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 582         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 583
 584         if (mrqc == 0) {
 585                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 586                         "supported", hf);
 587                 return;
 588         }
 589
 590         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 591 }
 592
 593 static void
 594 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 595 {
 596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 597         uint32_t i;
 598
 599         for (i = 0; i < nb_lport_new; i++) {
 600                 /* Set unicast mode by default. App can change
 601                  * to other mode in other API func.
 602                  */
 603                 fm10k_mbx_lock(hw);
 604                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 605                         FM10K_XCAST_MODE_NONE);
 606                 fm10k_mbx_unlock(hw);
 607         }
 608 }
 609
 610 static void
 611 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 612 {
 613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 615         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 616         struct fm10k_macvlan_filter_info *macvlan;
 617         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 618         uint16_t nb_lport_new;
 619
 620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 621         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 622
 623         fm10k_dev_rss_configure(dev);
 624
 625         /* only PF supports VMDQ */
 626         if (hw->mac.type != fm10k_mac_pf)
 627                 return;
 628
 629         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 630                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 631
 632         /* no pool number change, no need to update logic port and VLAN/MAC */
 633         if (macvlan->nb_queue_pools == nb_queue_pools)
 634                 return;
 635
 636         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 637         fm10k_dev_logic_port_update(dev, nb_lport_new);
 638
 639         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 640         memset(dev->data->mac_addrs, 0,
 641                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 642         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 643                 &dev->data->mac_addrs[0]);
 644         memset(macvlan, 0, sizeof(*macvlan));
 645         macvlan->nb_queue_pools = nb_queue_pools;
 646
 647         if (nb_queue_pools)
 648                 fm10k_dev_vmdq_rx_configure(dev);
 649         else
 650                 fm10k_dev_pf_main_vsi_reset(dev);
 651 }
 652
 653 static int
 654 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 655 {
 656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 657         int i, ret;
 658         struct fm10k_tx_queue *txq;
 659         uint64_t base_addr;
 660         uint32_t size;
 661
 662         /* Disable TXINT to avoid possible interrupt */
 663         for (i = 0; i < hw->mac.max_queues; i++)
 664                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 665                                 3 << FM10K_TXINT_TIMER_SHIFT);
 666
 667         /* Setup TX queue */
 668         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 669                 txq = dev->data->tx_queues[i];
 670                 base_addr = txq->hw_ring_phys_addr;
 671                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 672
 673                 /* disable queue to avoid issues while updating state */
 674                 ret = tx_queue_disable(hw, i);
 675                 if (ret) {
 676                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 677                         return -1;
 678                 }
 679                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 680                  * register is read-only for VF.
 681                  */
 682                 if (fm10k_check_ftag(dev->device->devargs)) {
 683                         if (hw->mac.type == fm10k_mac_pf) {
 684                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 685                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 686                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 687                         } else {
 688                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 689                                 return -ENOTSUP;
 690                         }
 691                 }
 692
 693                 /* set location and size for descriptor ring */
 694                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 695                                 base_addr & UINT64_LOWER_32BITS_MASK);
 696                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 697                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 698                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 699
 700                 /* assign default SGLORT for each TX queue by PF */
 701                 if (hw->mac.type == fm10k_mac_pf)
 702                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 703         }
 704
 705         /* set up vector or scalar TX function as appropriate */
 706         fm10k_set_tx_function(dev);
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         struct fm10k_macvlan_filter_info *macvlan;
 716         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 717         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 718         int i, ret;
 719         struct fm10k_rx_queue *rxq;
 720         uint64_t base_addr;
 721         uint32_t size;
 722         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 723         uint32_t logic_port = hw->mac.dglort_map;
 724         uint16_t buf_size;
 725         uint16_t queue_stride = 0;
 726
 727         /* enable RXINT for interrupt mode */
 728         i = 0;
 729         if (rte_intr_dp_is_en(intr_handle)) {
 730                 for (; i < dev->data->nb_rx_queues; i++) {
 731                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 732                         if (hw->mac.type == fm10k_mac_pf)
 733                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 734                                         FM10K_ITR_AUTOMASK |
 735                                         FM10K_ITR_MASK_CLEAR);
 736                         else
 737                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 738                                         FM10K_ITR_AUTOMASK |
 739                                         FM10K_ITR_MASK_CLEAR);
 740                 }
 741         }
 742         /* Disable other RXINT to avoid possible interrupt */
 743         for (; i < hw->mac.max_queues; i++)
 744                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 745                         3 << FM10K_RXINT_TIMER_SHIFT);
 746
 747         /* Setup RX queues */
 748         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 749                 rxq = dev->data->rx_queues[i];
 750                 base_addr = rxq->hw_ring_phys_addr;
 751                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 752
 753                 /* disable queue to avoid issues while updating state */
 754                 ret = rx_queue_disable(hw, i);
 755                 if (ret) {
 756                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 757                         return -1;
 758                 }
 759
 760                 /* Setup the Base and Length of the Rx Descriptor Ring */
 761                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 762                                 base_addr & UINT64_LOWER_32BITS_MASK);
 763                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 764                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 765                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 766
 767                 /* Configure the Rx buffer size for one buff without split */
 768                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 769                         RTE_PKTMBUF_HEADROOM);
 770                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 771                  * reserved for this purpose, and the worst case could be 511B.
 772                  * But SRR reg assumes all buffers have the same size. In order
 773                  * to fill the gap, we'll have to consider the worst case and
 774                  * assume 512B is reserved. If we don't do so, it's possible
 775                  * for HW to overwrite data to next mbuf.
 776                  */
 777                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 778
 779                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 780                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 781                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 782
 783                 /* It adds dual VLAN length for supporting dual VLAN */
 784                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 785                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 786                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 787                         uint32_t reg;
 788                         dev->data->scattered_rx = 1;
 789                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 790                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 791                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 792                 }
 793
 794                 /* Enable drop on empty, it's RO for VF */
 795                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 796                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 797
 798                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 799                 FM10K_WRITE_FLUSH(hw);
 800         }
 801
 802         /* Configure VMDQ/RSS if applicable */
 803         fm10k_dev_mq_rx_configure(dev);
 804
 805         /* Decide the best RX function */
 806         fm10k_set_rx_function(dev);
 807
 808         /* update RX_SGLORT for loopback suppress*/
 809         if (hw->mac.type != fm10k_mac_pf)
 810                 return 0;
 811         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 812         if (macvlan->nb_queue_pools)
 813                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 814         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 815                 if (i && queue_stride && !(i % queue_stride))
 816                         logic_port++;
 817                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 818         }
 819
 820         return 0;
 821 }
 822
 823 static int
 824 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 825 {
 826         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 827         int err = -1;
 828         uint32_t reg;
 829         struct fm10k_rx_queue *rxq;
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         if (rx_queue_id < dev->data->nb_rx_queues) {
 834                 rxq = dev->data->rx_queues[rx_queue_id];
 835                 err = rx_queue_reset(rxq);
 836                 if (err == -ENOMEM) {
 837                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 838                         return err;
 839                 } else if (err == -EINVAL) {
 840                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 841                                 " %d", err);
 842                         return err;
 843                 }
 844
 845                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 846                  * Note: this must be done AFTER the queue is enabled on real
 847                  * hardware, but BEFORE the queue is enabled when using the
 848                  * emulation platform. Do it in both places for now and remove
 849                  * this comment and the following two register writes when the
 850                  * emulation platform is no longer being used.
 851                  */
 852                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854
 855                 /* Set PF ownership flag for PF devices */
 856                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 857                 if (hw->mac.type == fm10k_mac_pf)
 858                         reg |= FM10K_RXQCTL_PF;
 859                 reg |= FM10K_RXQCTL_ENABLE;
 860                 /* enable RX queue */
 861                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 862                 FM10K_WRITE_FLUSH(hw);
 863
 864                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 865                  * Note: this must be done AFTER the queue is enabled
 866                  */
 867                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 868                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 869                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 870         }
 871
 872         return err;
 873 }
 874
 875 static int
 876 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (rx_queue_id < dev->data->nb_rx_queues) {
 883                 /* Disable RX queue */
 884                 rx_queue_disable(hw, rx_queue_id);
 885
 886                 /* Free mbuf and clean HW ring */
 887                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 888                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 896 {
 897         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         /** @todo - this should be defined in the shared code */
 899 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 900         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 901         int err = 0;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (tx_queue_id < dev->data->nb_tx_queues) {
 906                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 907
 908                 q->ops->reset(q);
 909
 910                 /* reset head and tail pointers */
 911                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 912                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 913
 914                 /* enable TX queue */
 915                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 916                                         FM10K_TXDCTL_ENABLE | txdctl);
 917                 FM10K_WRITE_FLUSH(hw);
 918                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 919         } else
 920                 err = -1;
 921
 922         return err;
 923 }
 924
 925 static int
 926 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 927 {
 928         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (tx_queue_id < dev->data->nb_tx_queues) {
 933                 tx_queue_disable(hw, tx_queue_id);
 934                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 935                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 936         }
 937
 938         return 0;
 939 }
 940
 941 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 942 {
 943         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 944                 != FM10K_DGLORTMAP_NONE);
 945 }
 946
 947 static void
 948 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 949 {
 950         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return;
 958
 959         fm10k_mbx_lock(hw);
 960         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 961                                 FM10K_XCAST_MODE_PROMISC);
 962         fm10k_mbx_unlock(hw);
 963
 964         if (status != FM10K_SUCCESS)
 965                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 966 }
 967
 968 static void
 969 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint8_t mode;
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return;
 980
 981         if (dev->data->all_multicast == 1)
 982                 mode = FM10K_XCAST_MODE_ALLMULTI;
 983         else
 984                 mode = FM10K_XCAST_MODE_NONE;
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 mode);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         /* If promiscuous mode is enabled, it doesn't make sense to enable
1008          * allmulticast and disable promiscuous since fm10k only can select
1009          * one of the modes.
1010          */
1011         if (dev->data->promiscuous) {
1012                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1013                         "needn't enable allmulticast");
1014                 return;
1015         }
1016
1017         fm10k_mbx_lock(hw);
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_ALLMULTI);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         int status;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Return if it didn't acquire valid glort range */
1035         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1036                 return;
1037
1038         if (dev->data->promiscuous) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1040                         "since promisc mode is enabled");
1041                 return;
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         /* Change mode to unicast mode */
1046         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1047                                 FM10K_XCAST_MODE_NONE);
1048         fm10k_mbx_unlock(hw);
1049
1050         if (status != FM10K_SUCCESS)
1051                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1052 }
1053
1054 static void
1055 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1056 {
1057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1059         uint16_t nb_queue_pools;
1060         struct fm10k_macvlan_filter_info *macvlan;
1061
1062         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1063         nb_queue_pools = macvlan->nb_queue_pools;
1064         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1065         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1066
1067         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1068         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1069         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         hw->mac.dglort_map;
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1072         /* Configure VMDQ/RSS DGlort Decoder */
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1074
1075         /* Flow Director configurations, only queue number is valid. */
1076         dglortdec = fls(dev->data->nb_rx_queues - 1);
1077         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1078                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1080         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1081
1082         /* Invalidate all other GLORT entries */
1083         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1084                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1085                                 FM10K_DGLORTMAP_NONE);
1086 }
1087
1088 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1089 static int
1090 fm10k_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         int i, diag;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* stop, init, then start the hw */
1098         diag = fm10k_stop_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_init_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_start_hw(hw);
1111         if (diag != FM10K_SUCCESS) {
1112                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1113                 return -EIO;
1114         }
1115
1116         diag = fm10k_dev_tx_init(dev);
1117         if (diag) {
1118                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1119                 return diag;
1120         }
1121
1122         if (fm10k_dev_rxq_interrupt_setup(dev))
1123                 return -EIO;
1124
1125         diag = fm10k_dev_rx_init(dev);
1126         if (diag) {
1127                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1128                 return diag;
1129         }
1130
1131         if (hw->mac.type == fm10k_mac_pf)
1132                 fm10k_dev_dglort_map_configure(dev);
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 struct fm10k_rx_queue *rxq;
1136                 rxq = dev->data->rx_queues[i];
1137
1138                 if (rxq->rx_deferred_start)
1139                         continue;
1140                 diag = fm10k_dev_rx_queue_start(dev, i);
1141                 if (diag != 0) {
1142                         int j;
1143                         for (j = 0; j < i; ++j)
1144                                 rx_queue_clean(dev->data->rx_queues[j]);
1145                         return diag;
1146                 }
1147         }
1148
1149         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1150                 struct fm10k_tx_queue *txq;
1151                 txq = dev->data->tx_queues[i];
1152
1153                 if (txq->tx_deferred_start)
1154                         continue;
1155                 diag = fm10k_dev_tx_queue_start(dev, i);
1156                 if (diag != 0) {
1157                         int j;
1158                         for (j = 0; j < i; ++j)
1159                                 tx_queue_clean(dev->data->tx_queues[j]);
1160                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1161                                 rx_queue_clean(dev->data->rx_queues[j]);
1162                         return diag;
1163                 }
1164         }
1165
1166         /* Update default vlan when not in VMDQ mode */
1167         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1168                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1169
1170         fm10k_link_update(dev, 0);
1171
1172         return 0;
1173 }
1174
1175 static void
1176 fm10k_dev_stop(struct rte_eth_dev *dev)
1177 {
1178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1179         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1180         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1181         int i;
1182
1183         PMD_INIT_FUNC_TRACE();
1184
1185         if (dev->data->tx_queues)
1186                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1187                         fm10k_dev_tx_queue_stop(dev, i);
1188
1189         if (dev->data->rx_queues)
1190                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1191                         fm10k_dev_rx_queue_stop(dev, i);
1192
1193         /* Disable datapath event */
1194         if (rte_intr_dp_is_en(intr_handle)) {
1195                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1196                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1197                                 3 << FM10K_RXINT_TIMER_SHIFT);
1198                         if (hw->mac.type == fm10k_mac_pf)
1199                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                         else
1202                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1203                                         FM10K_ITR_MASK_SET);
1204                 }
1205         }
1206         /* Clean datapath event and queue/vec mapping */
1207         rte_intr_efd_disable(intr_handle);
1208         rte_free(intr_handle->intr_vec);
1209         intr_handle->intr_vec = NULL;
1210 }
1211
1212 static void
1213 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1214 {
1215         int i;
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         if (dev->data->tx_queues) {
1220                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1222
1223                         tx_queue_free(txq);
1224                 }
1225         }
1226
1227         if (dev->data->rx_queues) {
1228                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1229                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1230         }
1231 }
1232
1233 static void
1234 fm10k_dev_close(struct rte_eth_dev *dev)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237
1238         PMD_INIT_FUNC_TRACE();
1239
1240         fm10k_mbx_lock(hw);
1241         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1242                 MAX_LPORT_NUM, false);
1243         fm10k_mbx_unlock(hw);
1244
1245         /* allow 100ms for device to quiesce */
1246         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1247
1248         /* Stop mailbox service first */
1249         fm10k_close_mbx_service(hw);
1250         fm10k_dev_stop(dev);
1251         fm10k_dev_queue_release(dev);
1252         fm10k_stop_hw(hw);
1253 }
1254
1255 static int
1256 fm10k_link_update(struct rte_eth_dev *dev,
1257         __rte_unused int wait_to_complete)
1258 {
1259         struct fm10k_dev_info *dev_info =
1260                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1261         PMD_INIT_FUNC_TRACE();
1262
1263         /* The speed is ~50Gbps per Gen3 x8 PCIe interface. For now, we
1264          * leave the speed undefined since there is no 50Gbps Ethernet.
1265          */
1266         dev->data->dev_link.link_speed  = 0;
1267         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1268         dev->data->dev_link.link_status =
1269                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1270
1271         return 0;
1272 }
1273
1274 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1275         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1276 {
1277         unsigned i, q;
1278         unsigned count = 0;
1279
1280         if (xstats_names != NULL) {
1281                 /* Note: limit checked in rte_eth_xstats_names() */
1282
1283                 /* Global stats */
1284                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1285                         snprintf(xstats_names[count].name,
1286                                 sizeof(xstats_names[count].name),
1287                                 "%s", fm10k_hw_stats_strings[count].name);
1288                         count++;
1289                 }
1290
1291                 /* PF queue stats */
1292                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1293                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1294                                 snprintf(xstats_names[count].name,
1295                                         sizeof(xstats_names[count].name),
1296                                         "rx_q%u_%s", q,
1297                                         fm10k_hw_stats_rx_q_strings[i].name);
1298                                 count++;
1299                         }
1300                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1301                                 snprintf(xstats_names[count].name,
1302                                         sizeof(xstats_names[count].name),
1303                                         "tx_q%u_%s", q,
1304                                         fm10k_hw_stats_tx_q_strings[i].name);
1305                                 count++;
1306                         }
1307                 }
1308         }
1309         return FM10K_NB_XSTATS;
1310 }
1311
1312 static int
1313 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1314                  unsigned n)
1315 {
1316         struct fm10k_hw_stats *hw_stats =
1317                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1318         unsigned i, q, count = 0;
1319
1320         if (n < FM10K_NB_XSTATS)
1321                 return FM10K_NB_XSTATS;
1322
1323         /* Global stats */
1324         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1325                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1326                         fm10k_hw_stats_strings[count].offset);
1327                 xstats[count].id = count;
1328                 count++;
1329         }
1330
1331         /* PF queue stats */
1332         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1333                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1334                         xstats[count].value =
1335                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1336                                 fm10k_hw_stats_rx_q_strings[i].offset);
1337                         xstats[count].id = count;
1338                         count++;
1339                 }
1340                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1341                         xstats[count].value =
1342                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1343                                 fm10k_hw_stats_tx_q_strings[i].offset);
1344                         xstats[count].id = count;
1345                         count++;
1346                 }
1347         }
1348
1349         return FM10K_NB_XSTATS;
1350 }
1351
1352 static int
1353 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1354 {
1355         uint64_t ipackets, opackets, ibytes, obytes;
1356         struct fm10k_hw *hw =
1357                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1358         struct fm10k_hw_stats *hw_stats =
1359                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1360         int i;
1361
1362         PMD_INIT_FUNC_TRACE();
1363
1364         fm10k_update_hw_stats(hw, hw_stats);
1365
1366         ipackets = opackets = ibytes = obytes = 0;
1367         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1368                 (i < hw->mac.max_queues); ++i) {
1369                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1370                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1371                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1372                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1373                 ipackets += stats->q_ipackets[i];
1374                 opackets += stats->q_opackets[i];
1375                 ibytes   += stats->q_ibytes[i];
1376                 obytes   += stats->q_obytes[i];
1377         }
1378         stats->ipackets = ipackets;
1379         stats->opackets = opackets;
1380         stats->ibytes = ibytes;
1381         stats->obytes = obytes;
1382         return 0;
1383 }
1384
1385 static void
1386 fm10k_stats_reset(struct rte_eth_dev *dev)
1387 {
1388         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1389         struct fm10k_hw_stats *hw_stats =
1390                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1391
1392         PMD_INIT_FUNC_TRACE();
1393
1394         memset(hw_stats, 0, sizeof(*hw_stats));
1395         fm10k_rebind_hw_stats(hw, hw_stats);
1396 }
1397
1398 static void
1399 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1400         struct rte_eth_dev_info *dev_info)
1401 {
1402         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1403         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1404
1405         PMD_INIT_FUNC_TRACE();
1406
1407         dev_info->pci_dev            = pdev;
1408         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1409         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1410         dev_info->max_rx_queues      = hw->mac.max_queues;
1411         dev_info->max_tx_queues      = hw->mac.max_queues;
1412         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1413         dev_info->max_hash_mac_addrs = 0;
1414         dev_info->max_vfs            = pdev->max_vfs;
1415         dev_info->vmdq_pool_base     = 0;
1416         dev_info->vmdq_queue_base    = 0;
1417         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1418         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1419         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1420         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1421                                     dev_info->rx_queue_offload_capa;
1422         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1423         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1424                                     dev_info->tx_queue_offload_capa;
1425
1426         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1427         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1428
1429         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1430                 .rx_thresh = {
1431                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1432                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1433                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1434                 },
1435                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1436                 .rx_drop_en = 0,
1437                 .offloads = 0,
1438         };
1439
1440         dev_info->default_txconf = (struct rte_eth_txconf) {
1441                 .tx_thresh = {
1442                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1443                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1444                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1445                 },
1446                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1447                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1448                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1449                 .offloads = 0,
1450         };
1451
1452         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1453                 .nb_max = FM10K_MAX_RX_DESC,
1454                 .nb_min = FM10K_MIN_RX_DESC,
1455                 .nb_align = FM10K_MULT_RX_DESC,
1456         };
1457
1458         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1459                 .nb_max = FM10K_MAX_TX_DESC,
1460                 .nb_min = FM10K_MIN_TX_DESC,
1461                 .nb_align = FM10K_MULT_TX_DESC,
1462                 .nb_seg_max = FM10K_TX_MAX_SEG,
1463                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1464         };
1465
1466         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1467                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1468                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1469 }
1470
1471 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1472 static const uint32_t *
1473 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1474 {
1475         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1476             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1477                 static uint32_t ptypes[] = {
1478                         /* refers to rx_desc_to_ol_flags() */
1479                         RTE_PTYPE_L2_ETHER,
1480                         RTE_PTYPE_L3_IPV4,
1481                         RTE_PTYPE_L3_IPV4_EXT,
1482                         RTE_PTYPE_L3_IPV6,
1483                         RTE_PTYPE_L3_IPV6_EXT,
1484                         RTE_PTYPE_L4_TCP,
1485                         RTE_PTYPE_L4_UDP,
1486                         RTE_PTYPE_UNKNOWN
1487                 };
1488
1489                 return ptypes;
1490         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1491                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1492                 static uint32_t ptypes_vec[] = {
1493                         /* refers to fm10k_desc_to_pktype_v() */
1494                         RTE_PTYPE_L3_IPV4,
1495                         RTE_PTYPE_L3_IPV4_EXT,
1496                         RTE_PTYPE_L3_IPV6,
1497                         RTE_PTYPE_L3_IPV6_EXT,
1498                         RTE_PTYPE_L4_TCP,
1499                         RTE_PTYPE_L4_UDP,
1500                         RTE_PTYPE_TUNNEL_GENEVE,
1501                         RTE_PTYPE_TUNNEL_NVGRE,
1502                         RTE_PTYPE_TUNNEL_VXLAN,
1503                         RTE_PTYPE_TUNNEL_GRE,
1504                         RTE_PTYPE_UNKNOWN
1505                 };
1506
1507                 return ptypes_vec;
1508         }
1509
1510         return NULL;
1511 }
1512 #else
1513 static const uint32_t *
1514 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1515 {
1516         return NULL;
1517 }
1518 #endif
1519
1520 static int
1521 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1522 {
1523         s32 result;
1524         uint16_t mac_num = 0;
1525         uint32_t vid_idx, vid_bit, mac_index;
1526         struct fm10k_hw *hw;
1527         struct fm10k_macvlan_filter_info *macvlan;
1528         struct rte_eth_dev_data *data = dev->data;
1529
1530         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1531         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1532
1533         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1534                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1535                 return -EINVAL;
1536         }
1537
1538         if (vlan_id > ETH_VLAN_ID_MAX) {
1539                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1540                 return -EINVAL;
1541         }
1542
1543         vid_idx = FM10K_VFTA_IDX(vlan_id);
1544         vid_bit = FM10K_VFTA_BIT(vlan_id);
1545         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1546         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1547                 return 0;
1548         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1549         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1550                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1551                         "in the VLAN filter table");
1552                 return -EINVAL;
1553         }
1554
1555         fm10k_mbx_lock(hw);
1556         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1557         fm10k_mbx_unlock(hw);
1558         if (result != FM10K_SUCCESS) {
1559                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1560                 return -EIO;
1561         }
1562
1563         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1564                         (result == FM10K_SUCCESS); mac_index++) {
1565                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1566                         continue;
1567                 if (mac_num > macvlan->mac_num - 1) {
1568                         PMD_INIT_LOG(ERR, "MAC address number "
1569                                         "not match");
1570                         break;
1571                 }
1572                 fm10k_mbx_lock(hw);
1573                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1574                         data->mac_addrs[mac_index].addr_bytes,
1575                         vlan_id, on, 0);
1576                 fm10k_mbx_unlock(hw);
1577                 mac_num++;
1578         }
1579         if (result != FM10K_SUCCESS) {
1580                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1581                 return -EIO;
1582         }
1583
1584         if (on) {
1585                 macvlan->vlan_num++;
1586                 macvlan->vfta[vid_idx] |= vid_bit;
1587         } else {
1588                 macvlan->vlan_num--;
1589                 macvlan->vfta[vid_idx] &= ~vid_bit;
1590         }
1591         return 0;
1592 }
1593
1594 static int
1595 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1596 {
1597         if (mask & ETH_VLAN_STRIP_MASK) {
1598                 if (!(dev->data->dev_conf.rxmode.offloads &
1599                         DEV_RX_OFFLOAD_VLAN_STRIP))
1600                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1601                                         "always on in fm10k");
1602         }
1603
1604         if (mask & ETH_VLAN_EXTEND_MASK) {
1605                 if (dev->data->dev_conf.rxmode.offloads &
1606                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1607                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1608                                         "supported in fm10k");
1609         }
1610
1611         if (mask & ETH_VLAN_FILTER_MASK) {
1612                 if (!(dev->data->dev_conf.rxmode.offloads &
1613                         DEV_RX_OFFLOAD_VLAN_FILTER))
1614                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1615         }
1616
1617         return 0;
1618 }
1619
1620 /* Add/Remove a MAC address, and update filters to main VSI */
1621 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1622                 const u8 *mac, bool add, uint32_t pool)
1623 {
1624         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1625         struct fm10k_macvlan_filter_info *macvlan;
1626         uint32_t i, j, k;
1627
1628         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1629
1630         if (pool != MAIN_VSI_POOL_NUMBER) {
1631                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1632                         "mac to pool %u", pool);
1633                 return;
1634         }
1635         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1636                 if (!macvlan->vfta[j])
1637                         continue;
1638                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1639                         if (!(macvlan->vfta[j] & (1 << k)))
1640                                 continue;
1641                         if (i + 1 > macvlan->vlan_num) {
1642                                 PMD_INIT_LOG(ERR, "vlan number not match");
1643                                 return;
1644                         }
1645                         fm10k_mbx_lock(hw);
1646                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1647                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1648                         fm10k_mbx_unlock(hw);
1649                         i++;
1650                 }
1651         }
1652 }
1653
1654 /* Add/Remove a MAC address, and update filters to VMDQ */
1655 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1656                 const u8 *mac, bool add, uint32_t pool)
1657 {
1658         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1659         struct fm10k_macvlan_filter_info *macvlan;
1660         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1661         uint32_t i;
1662
1663         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1664         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1665
1666         if (pool > macvlan->nb_queue_pools) {
1667                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1668                         " Max pool is %u",
1669                         pool, macvlan->nb_queue_pools);
1670                 return;
1671         }
1672         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1673                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1674                         continue;
1675                 fm10k_mbx_lock(hw);
1676                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1677                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1678                 fm10k_mbx_unlock(hw);
1679         }
1680 }
1681
1682 /* Add/Remove a MAC address, and update filters */
1683 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1684                 const u8 *mac, bool add, uint32_t pool)
1685 {
1686         struct fm10k_macvlan_filter_info *macvlan;
1687
1688         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1689
1690         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1691                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1692         else
1693                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1694
1695         if (add)
1696                 macvlan->mac_num++;
1697         else
1698                 macvlan->mac_num--;
1699 }
1700
1701 /* Add a MAC address, and update filters */
1702 static int
1703 fm10k_macaddr_add(struct rte_eth_dev *dev,
1704                 struct ether_addr *mac_addr,
1705                 uint32_t index,
1706                 uint32_t pool)
1707 {
1708         struct fm10k_macvlan_filter_info *macvlan;
1709
1710         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1711         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1712         macvlan->mac_vmdq_id[index] = pool;
1713         return 0;
1714 }
1715
1716 /* Remove a MAC address, and update filters */
1717 static void
1718 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1719 {
1720         struct rte_eth_dev_data *data = dev->data;
1721         struct fm10k_macvlan_filter_info *macvlan;
1722
1723         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1724         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1725                         FALSE, macvlan->mac_vmdq_id[index]);
1726         macvlan->mac_vmdq_id[index] = 0;
1727 }
1728
1729 static inline int
1730 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1731 {
1732         if ((request < min) || (request > max) || ((request % mult) != 0))
1733                 return -1;
1734         else
1735                 return 0;
1736 }
1737
1738
1739 static inline int
1740 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1741 {
1742         if ((request < min) || (request > max) || ((div % request) != 0))
1743                 return -1;
1744         else
1745                 return 0;
1746 }
1747
1748 static inline int
1749 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1750 {
1751         uint16_t rx_free_thresh;
1752
1753         if (conf->rx_free_thresh == 0)
1754                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1755         else
1756                 rx_free_thresh = conf->rx_free_thresh;
1757
1758         /* make sure the requested threshold satisfies the constraints */
1759         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1760                         FM10K_RX_FREE_THRESH_MAX(q),
1761                         FM10K_RX_FREE_THRESH_DIV(q),
1762                         rx_free_thresh)) {
1763                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1764                         "less than or equal to %u, "
1765                         "greater than or equal to %u, "
1766                         "and a divisor of %u",
1767                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1768                         FM10K_RX_FREE_THRESH_MIN(q),
1769                         FM10K_RX_FREE_THRESH_DIV(q));
1770                 return -EINVAL;
1771         }
1772
1773         q->alloc_thresh = rx_free_thresh;
1774         q->drop_en = conf->rx_drop_en;
1775         q->rx_deferred_start = conf->rx_deferred_start;
1776
1777         return 0;
1778 }
1779
1780 /*
1781  * Hardware requires specific alignment for Rx packet buffers. At
1782  * least one of the following two conditions must be satisfied.
1783  *  1. Address is 512B aligned
1784  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1785  *
1786  * As such, the driver may need to adjust the DMA address within the
1787  * buffer by up to 512B.
1788  *
1789  * return 1 if the element size is valid, otherwise return 0.
1790  */
1791 static int
1792 mempool_element_size_valid(struct rte_mempool *mp)
1793 {
1794         uint32_t min_size;
1795
1796         /* elt_size includes mbuf header and headroom */
1797         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1798                         RTE_PKTMBUF_HEADROOM;
1799
1800         /* account for up to 512B of alignment */
1801         min_size -= FM10K_RX_DATABUF_ALIGN;
1802
1803         /* sanity check for overflow */
1804         if (min_size > mp->elt_size)
1805                 return 0;
1806
1807         /* size is valid */
1808         return 1;
1809 }
1810
1811 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1812 {
1813         RTE_SET_USED(dev);
1814
1815         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1816 }
1817
1818 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1819 {
1820         RTE_SET_USED(dev);
1821
1822         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1823                            DEV_RX_OFFLOAD_VLAN_FILTER |
1824                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1825                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1826                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1827                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1828                            DEV_RX_OFFLOAD_CRC_STRIP   |
1829                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1830 }
1831
1832 static int
1833 fm10k_check_rx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
1834 {
1835         uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
1836         uint64_t queue_supported = fm10k_get_rx_queue_offloads_capa(dev);
1837         uint64_t port_supported = fm10k_get_rx_port_offloads_capa(dev);
1838
1839         if ((requested & (queue_supported | port_supported)) != requested)
1840                 return 0;
1841
1842         if ((port_offloads ^ requested) & port_supported)
1843                 return 0;
1844
1845         return 1;
1846 }
1847
1848 static int
1849 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1850         uint16_t nb_desc, unsigned int socket_id,
1851         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1852 {
1853         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1854         struct fm10k_dev_info *dev_info =
1855                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1856         struct fm10k_rx_queue *q;
1857         const struct rte_memzone *mz;
1858
1859         PMD_INIT_FUNC_TRACE();
1860
1861         if (!fm10k_check_rx_queue_offloads(dev, conf->offloads)) {
1862                 PMD_INIT_LOG(ERR, "%p: Rx queue offloads 0x%" PRIx64
1863                         " don't match port offloads 0x%" PRIx64
1864                         " or supported port offloads 0x%" PRIx64
1865                         " or supported queue offloads 0x%" PRIx64,
1866                         (void *)dev, conf->offloads,
1867                         dev->data->dev_conf.rxmode.offloads,
1868                         fm10k_get_rx_port_offloads_capa(dev),
1869                         fm10k_get_rx_queue_offloads_capa(dev));
1870                 return -ENOTSUP;
1871         }
1872
1873         /* make sure the mempool element size can account for alignment. */
1874         if (!mempool_element_size_valid(mp)) {
1875                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1876                 return -EINVAL;
1877         }
1878
1879         /* make sure a valid number of descriptors have been requested */
1880         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1881                                 FM10K_MULT_RX_DESC, nb_desc)) {
1882                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1883                         "less than or equal to %"PRIu32", "
1884                         "greater than or equal to %u, "
1885                         "and a multiple of %u",
1886                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1887                         FM10K_MULT_RX_DESC);
1888                 return -EINVAL;
1889         }
1890
1891         /*
1892          * if this queue existed already, free the associated memory. The
1893          * queue cannot be reused in case we need to allocate memory on
1894          * different socket than was previously used.
1895          */
1896         if (dev->data->rx_queues[queue_id] != NULL) {
1897                 rx_queue_free(dev->data->rx_queues[queue_id]);
1898                 dev->data->rx_queues[queue_id] = NULL;
1899         }
1900
1901         /* allocate memory for the queue structure */
1902         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1903                                 socket_id);
1904         if (q == NULL) {
1905                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1906                 return -ENOMEM;
1907         }
1908
1909         /* setup queue */
1910         q->mp = mp;
1911         q->nb_desc = nb_desc;
1912         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1913         q->port_id = dev->data->port_id;
1914         q->queue_id = queue_id;
1915         q->tail_ptr = (volatile uint32_t *)
1916                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1917         q->offloads = conf->offloads;
1918         if (handle_rxconf(q, conf))
1919                 return -EINVAL;
1920
1921         /* allocate memory for the software ring */
1922         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1923                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1924                         RTE_CACHE_LINE_SIZE, socket_id);
1925         if (q->sw_ring == NULL) {
1926                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1927                 rte_free(q);
1928                 return -ENOMEM;
1929         }
1930
1931         /*
1932          * allocate memory for the hardware descriptor ring. A memzone large
1933          * enough to hold the maximum ring size is requested to allow for
1934          * resizing in later calls to the queue setup function.
1935          */
1936         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1937                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1938                                       socket_id);
1939         if (mz == NULL) {
1940                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1941                 rte_free(q->sw_ring);
1942                 rte_free(q);
1943                 return -ENOMEM;
1944         }
1945         q->hw_ring = mz->addr;
1946         q->hw_ring_phys_addr = mz->iova;
1947
1948         /* Check if number of descs satisfied Vector requirement */
1949         if (!rte_is_power_of_2(nb_desc)) {
1950                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1951                                     "preconditions - canceling the feature for "
1952                                     "the whole port[%d]",
1953                              q->queue_id, q->port_id);
1954                 dev_info->rx_vec_allowed = false;
1955         } else
1956                 fm10k_rxq_vec_setup(q);
1957
1958         dev->data->rx_queues[queue_id] = q;
1959         return 0;
1960 }
1961
1962 static void
1963 fm10k_rx_queue_release(void *queue)
1964 {
1965         PMD_INIT_FUNC_TRACE();
1966
1967         rx_queue_free(queue);
1968 }
1969
1970 static inline int
1971 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1972 {
1973         uint16_t tx_free_thresh;
1974         uint16_t tx_rs_thresh;
1975
1976         /* constraint MACROs require that tx_free_thresh is configured
1977          * before tx_rs_thresh */
1978         if (conf->tx_free_thresh == 0)
1979                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1980         else
1981                 tx_free_thresh = conf->tx_free_thresh;
1982
1983         /* make sure the requested threshold satisfies the constraints */
1984         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1985                         FM10K_TX_FREE_THRESH_MAX(q),
1986                         FM10K_TX_FREE_THRESH_DIV(q),
1987                         tx_free_thresh)) {
1988                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1989                         "less than or equal to %u, "
1990                         "greater than or equal to %u, "
1991                         "and a divisor of %u",
1992                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1993                         FM10K_TX_FREE_THRESH_MIN(q),
1994                         FM10K_TX_FREE_THRESH_DIV(q));
1995                 return -EINVAL;
1996         }
1997
1998         q->free_thresh = tx_free_thresh;
1999
2000         if (conf->tx_rs_thresh == 0)
2001                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
2002         else
2003                 tx_rs_thresh = conf->tx_rs_thresh;
2004
2005         q->tx_deferred_start = conf->tx_deferred_start;
2006
2007         /* make sure the requested threshold satisfies the constraints */
2008         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
2009                         FM10K_TX_RS_THRESH_MAX(q),
2010                         FM10K_TX_RS_THRESH_DIV(q),
2011                         tx_rs_thresh)) {
2012                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
2013                         "less than or equal to %u, "
2014                         "greater than or equal to %u, "
2015                         "and a divisor of %u",
2016                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
2017                         FM10K_TX_RS_THRESH_MIN(q),
2018                         FM10K_TX_RS_THRESH_DIV(q));
2019                 return -EINVAL;
2020         }
2021
2022         q->rs_thresh = tx_rs_thresh;
2023
2024         return 0;
2025 }
2026
2027 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
2028 {
2029         RTE_SET_USED(dev);
2030
2031         return 0;
2032 }
2033
2034 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
2035 {
2036         RTE_SET_USED(dev);
2037
2038         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
2039                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
2040                           DEV_TX_OFFLOAD_UDP_CKSUM   |
2041                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2042                           DEV_TX_OFFLOAD_TCP_TSO);
2043 }
2044
2045 static int
2046 fm10k_check_tx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
2047 {
2048         uint64_t port_offloads = dev->data->dev_conf.txmode.offloads;
2049         uint64_t queue_supported = fm10k_get_tx_queue_offloads_capa(dev);
2050         uint64_t port_supported = fm10k_get_tx_port_offloads_capa(dev);
2051
2052         if ((requested & (queue_supported | port_supported)) != requested)
2053                 return 0;
2054
2055         if ((port_offloads ^ requested) & port_supported)
2056                 return 0;
2057
2058         return 1;
2059 }
2060
2061 static int
2062 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2063         uint16_t nb_desc, unsigned int socket_id,
2064         const struct rte_eth_txconf *conf)
2065 {
2066         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2067         struct fm10k_tx_queue *q;
2068         const struct rte_memzone *mz;
2069
2070         PMD_INIT_FUNC_TRACE();
2071
2072         if (!fm10k_check_tx_queue_offloads(dev, conf->offloads)) {
2073                 PMD_INIT_LOG(ERR, "%p: Tx queue offloads 0x%" PRIx64
2074                         " don't match port offloads 0x%" PRIx64
2075                         " or supported port offloads 0x%" PRIx64
2076                         " or supported queue offloads 0x%" PRIx64,
2077                         (void *)dev, conf->offloads,
2078                         dev->data->dev_conf.txmode.offloads,
2079                         fm10k_get_tx_port_offloads_capa(dev),
2080                         fm10k_get_tx_queue_offloads_capa(dev));
2081                 return -ENOTSUP;
2082         }
2083
2084         /* make sure a valid number of descriptors have been requested */
2085         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2086                                 FM10K_MULT_TX_DESC, nb_desc)) {
2087                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2088                         "less than or equal to %"PRIu32", "
2089                         "greater than or equal to %u, "
2090                         "and a multiple of %u",
2091                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2092                         FM10K_MULT_TX_DESC);
2093                 return -EINVAL;
2094         }
2095
2096         /*
2097          * if this queue existed already, free the associated memory. The
2098          * queue cannot be reused in case we need to allocate memory on
2099          * different socket than was previously used.
2100          */
2101         if (dev->data->tx_queues[queue_id] != NULL) {
2102                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2103
2104                 tx_queue_free(txq);
2105                 dev->data->tx_queues[queue_id] = NULL;
2106         }
2107
2108         /* allocate memory for the queue structure */
2109         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2110                                 socket_id);
2111         if (q == NULL) {
2112                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2113                 return -ENOMEM;
2114         }
2115
2116         /* setup queue */
2117         q->nb_desc = nb_desc;
2118         q->port_id = dev->data->port_id;
2119         q->queue_id = queue_id;
2120         q->txq_flags = conf->txq_flags;
2121         q->offloads = conf->offloads;
2122         q->ops = &def_txq_ops;
2123         q->tail_ptr = (volatile uint32_t *)
2124                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2125         if (handle_txconf(q, conf))
2126                 return -EINVAL;
2127
2128         /* allocate memory for the software ring */
2129         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2130                                         nb_desc * sizeof(struct rte_mbuf *),
2131                                         RTE_CACHE_LINE_SIZE, socket_id);
2132         if (q->sw_ring == NULL) {
2133                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2134                 rte_free(q);
2135                 return -ENOMEM;
2136         }
2137
2138         /*
2139          * allocate memory for the hardware descriptor ring. A memzone large
2140          * enough to hold the maximum ring size is requested to allow for
2141          * resizing in later calls to the queue setup function.
2142          */
2143         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2144                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2145                                       socket_id);
2146         if (mz == NULL) {
2147                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2148                 rte_free(q->sw_ring);
2149                 rte_free(q);
2150                 return -ENOMEM;
2151         }
2152         q->hw_ring = mz->addr;
2153         q->hw_ring_phys_addr = mz->iova;
2154
2155         /*
2156          * allocate memory for the RS bit tracker. Enough slots to hold the
2157          * descriptor index for each RS bit needing to be set are required.
2158          */
2159         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2160                                 ((nb_desc + 1) / q->rs_thresh) *
2161                                 sizeof(uint16_t),
2162                                 RTE_CACHE_LINE_SIZE, socket_id);
2163         if (q->rs_tracker.list == NULL) {
2164                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2165                 rte_free(q->sw_ring);
2166                 rte_free(q);
2167                 return -ENOMEM;
2168         }
2169
2170         dev->data->tx_queues[queue_id] = q;
2171         return 0;
2172 }
2173
2174 static void
2175 fm10k_tx_queue_release(void *queue)
2176 {
2177         struct fm10k_tx_queue *q = queue;
2178         PMD_INIT_FUNC_TRACE();
2179
2180         tx_queue_free(q);
2181 }
2182
2183 static int
2184 fm10k_reta_update(struct rte_eth_dev *dev,
2185                         struct rte_eth_rss_reta_entry64 *reta_conf,
2186                         uint16_t reta_size)
2187 {
2188         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2189         uint16_t i, j, idx, shift;
2190         uint8_t mask;
2191         uint32_t reta;
2192
2193         PMD_INIT_FUNC_TRACE();
2194
2195         if (reta_size > FM10K_MAX_RSS_INDICES) {
2196                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2197                         "(%d) doesn't match the number hardware can supported "
2198                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2199                 return -EINVAL;
2200         }
2201
2202         /*
2203          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2204          * 128-entries in 32 registers
2205          */
2206         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2207                 idx = i / RTE_RETA_GROUP_SIZE;
2208                 shift = i % RTE_RETA_GROUP_SIZE;
2209                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2210                                 BIT_MASK_PER_UINT32);
2211                 if (mask == 0)
2212                         continue;
2213
2214                 reta = 0;
2215                 if (mask != BIT_MASK_PER_UINT32)
2216                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2217
2218                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2219                         if (mask & (0x1 << j)) {
2220                                 if (mask != 0xF)
2221                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2222                                 reta |= reta_conf[idx].reta[shift + j] <<
2223                                                 (CHAR_BIT * j);
2224                         }
2225                 }
2226                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2227         }
2228
2229         return 0;
2230 }
2231
2232 static int
2233 fm10k_reta_query(struct rte_eth_dev *dev,
2234                         struct rte_eth_rss_reta_entry64 *reta_conf,
2235                         uint16_t reta_size)
2236 {
2237         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2238         uint16_t i, j, idx, shift;
2239         uint8_t mask;
2240         uint32_t reta;
2241
2242         PMD_INIT_FUNC_TRACE();
2243
2244         if (reta_size < FM10K_MAX_RSS_INDICES) {
2245                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2246                         "(%d) doesn't match the number hardware can supported "
2247                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2248                 return -EINVAL;
2249         }
2250
2251         /*
2252          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2253          * 128-entries in 32 registers
2254          */
2255         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2256                 idx = i / RTE_RETA_GROUP_SIZE;
2257                 shift = i % RTE_RETA_GROUP_SIZE;
2258                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2259                                 BIT_MASK_PER_UINT32);
2260                 if (mask == 0)
2261                         continue;
2262
2263                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2264                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2265                         if (mask & (0x1 << j))
2266                                 reta_conf[idx].reta[shift + j] = ((reta >>
2267                                         CHAR_BIT * j) & UINT8_MAX);
2268                 }
2269         }
2270
2271         return 0;
2272 }
2273
2274 static int
2275 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2276         struct rte_eth_rss_conf *rss_conf)
2277 {
2278         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2279         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2280         uint32_t mrqc;
2281         uint64_t hf = rss_conf->rss_hf;
2282         int i;
2283
2284         PMD_INIT_FUNC_TRACE();
2285
2286         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2287                                 FM10K_RSSRK_ENTRIES_PER_REG))
2288                 return -EINVAL;
2289
2290         if (hf == 0)
2291                 return -EINVAL;
2292
2293         mrqc = 0;
2294         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2295         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2296         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2297         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2298         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2299         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2300         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2301         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2302         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2303
2304         /* If the mapping doesn't fit any supported, return */
2305         if (mrqc == 0)
2306                 return -EINVAL;
2307
2308         if (key != NULL)
2309                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2310                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2311
2312         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2313
2314         return 0;
2315 }
2316
2317 static int
2318 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2319         struct rte_eth_rss_conf *rss_conf)
2320 {
2321         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2322         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2323         uint32_t mrqc;
2324         uint64_t hf;
2325         int i;
2326
2327         PMD_INIT_FUNC_TRACE();
2328
2329         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2330                                 FM10K_RSSRK_ENTRIES_PER_REG))
2331                 return -EINVAL;
2332
2333         if (key != NULL)
2334                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2335                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2336
2337         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2338         hf = 0;
2339         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2340         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2341         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2342         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2343         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2344         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2345         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2346         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2347         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2348
2349         rss_conf->rss_hf = hf;
2350
2351         return 0;
2352 }
2353
2354 static void
2355 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2356 {
2357         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2358         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2359
2360         /* Bind all local non-queue interrupt to vector 0 */
2361         int_map |= FM10K_MISC_VEC_ID;
2362
2363         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2364         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2365         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2366         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2367         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2368         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2369
2370         /* Enable misc causes */
2371         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2372                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2373                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2374                                 FM10K_EIMR_ENABLE(MAILBOX) |
2375                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2376                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2377                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2378                                 FM10K_EIMR_ENABLE(VFLR));
2379
2380         /* Enable ITR 0 */
2381         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2382                                         FM10K_ITR_MASK_CLEAR);
2383         FM10K_WRITE_FLUSH(hw);
2384 }
2385
2386 static void
2387 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2388 {
2389         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2390         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2391
2392         int_map |= FM10K_MISC_VEC_ID;
2393
2394         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2395         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2396         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2397         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2398         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2399         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2400
2401         /* Disable misc causes */
2402         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2403                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2404                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2405                                 FM10K_EIMR_DISABLE(MAILBOX) |
2406                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2407                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2408                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2409                                 FM10K_EIMR_DISABLE(VFLR));
2410
2411         /* Disable ITR 0 */
2412         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2413         FM10K_WRITE_FLUSH(hw);
2414 }
2415
2416 static void
2417 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2418 {
2419         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2420         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2421
2422         /* Bind all local non-queue interrupt to vector 0 */
2423         int_map |= FM10K_MISC_VEC_ID;
2424
2425         /* Only INT 0 available, other 15 are reserved. */
2426         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2427
2428         /* Enable ITR 0 */
2429         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2430                                         FM10K_ITR_MASK_CLEAR);
2431         FM10K_WRITE_FLUSH(hw);
2432 }
2433
2434 static void
2435 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2436 {
2437         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2438         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2439
2440         int_map |= FM10K_MISC_VEC_ID;
2441
2442         /* Only INT 0 available, other 15 are reserved. */
2443         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2444
2445         /* Disable ITR 0 */
2446         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2447         FM10K_WRITE_FLUSH(hw);
2448 }
2449
2450 static int
2451 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2452 {
2453         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2454         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2455
2456         /* Enable ITR */
2457         if (hw->mac.type == fm10k_mac_pf)
2458                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2459                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2460         else
2461                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2462                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2463         rte_intr_enable(&pdev->intr_handle);
2464         return 0;
2465 }
2466
2467 static int
2468 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2469 {
2470         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2471         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2472
2473         /* Disable ITR */
2474         if (hw->mac.type == fm10k_mac_pf)
2475                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2476                         FM10K_ITR_MASK_SET);
2477         else
2478                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2479                         FM10K_ITR_MASK_SET);
2480         return 0;
2481 }
2482
2483 static int
2484 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2485 {
2486         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2487         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2488         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2489         uint32_t intr_vector, vec;
2490         uint16_t queue_id;
2491         int result = 0;
2492
2493         /* fm10k needs one separate interrupt for mailbox,
2494          * so only drivers which support multiple interrupt vectors
2495          * e.g. vfio-pci can work for fm10k interrupt mode
2496          */
2497         if (!rte_intr_cap_multiple(intr_handle) ||
2498                         dev->data->dev_conf.intr_conf.rxq == 0)
2499                 return result;
2500
2501         intr_vector = dev->data->nb_rx_queues;
2502
2503         /* disable interrupt first */
2504         rte_intr_disable(intr_handle);
2505         if (hw->mac.type == fm10k_mac_pf)
2506                 fm10k_dev_disable_intr_pf(dev);
2507         else
2508                 fm10k_dev_disable_intr_vf(dev);
2509
2510         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2511                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2512                 result = -EIO;
2513         }
2514
2515         if (rte_intr_dp_is_en(intr_handle) && !result) {
2516                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2517                         dev->data->nb_rx_queues * sizeof(int), 0);
2518                 if (intr_handle->intr_vec) {
2519                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2520                                         queue_id < dev->data->nb_rx_queues;
2521                                         queue_id++) {
2522                                 intr_handle->intr_vec[queue_id] = vec;
2523                                 if (vec < intr_handle->nb_efd - 1
2524                                                 + FM10K_RX_VEC_START)
2525                                         vec++;
2526                         }
2527                 } else {
2528                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2529                                 " intr_vec", dev->data->nb_rx_queues);
2530                         rte_intr_efd_disable(intr_handle);
2531                         result = -ENOMEM;
2532                 }
2533         }
2534
2535         if (hw->mac.type == fm10k_mac_pf)
2536                 fm10k_dev_enable_intr_pf(dev);
2537         else
2538                 fm10k_dev_enable_intr_vf(dev);
2539         rte_intr_enable(intr_handle);
2540         hw->mac.ops.update_int_moderator(hw);
2541         return result;
2542 }
2543
2544 static int
2545 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2546 {
2547         struct fm10k_fault fault;
2548         int err;
2549         const char *estr = "Unknown error";
2550
2551         /* Process PCA fault */
2552         if (eicr & FM10K_EICR_PCA_FAULT) {
2553                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2554                 if (err)
2555                         goto error;
2556                 switch (fault.type) {
2557                 case PCA_NO_FAULT:
2558                         estr = "PCA_NO_FAULT"; break;
2559                 case PCA_UNMAPPED_ADDR:
2560                         estr = "PCA_UNMAPPED_ADDR"; break;
2561                 case PCA_BAD_QACCESS_PF:
2562                         estr = "PCA_BAD_QACCESS_PF"; break;
2563                 case PCA_BAD_QACCESS_VF:
2564                         estr = "PCA_BAD_QACCESS_VF"; break;
2565                 case PCA_MALICIOUS_REQ:
2566                         estr = "PCA_MALICIOUS_REQ"; break;
2567                 case PCA_POISONED_TLP:
2568                         estr = "PCA_POISONED_TLP"; break;
2569                 case PCA_TLP_ABORT:
2570                         estr = "PCA_TLP_ABORT"; break;
2571                 default:
2572                         goto error;
2573                 }
2574                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2575                         estr, fault.func ? "VF" : "PF", fault.func,
2576                         fault.address, fault.specinfo);
2577         }
2578
2579         /* Process THI fault */
2580         if (eicr & FM10K_EICR_THI_FAULT) {
2581                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2582                 if (err)
2583                         goto error;
2584                 switch (fault.type) {
2585                 case THI_NO_FAULT:
2586                         estr = "THI_NO_FAULT"; break;
2587                 case THI_MAL_DIS_Q_FAULT:
2588                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2589                 default:
2590                         goto error;
2591                 }
2592                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2593                         estr, fault.func ? "VF" : "PF", fault.func,
2594                         fault.address, fault.specinfo);
2595         }
2596
2597         /* Process FUM fault */
2598         if (eicr & FM10K_EICR_FUM_FAULT) {
2599                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2600                 if (err)
2601                         goto error;
2602                 switch (fault.type) {
2603                 case FUM_NO_FAULT:
2604                         estr = "FUM_NO_FAULT"; break;
2605                 case FUM_UNMAPPED_ADDR:
2606                         estr = "FUM_UNMAPPED_ADDR"; break;
2607                 case FUM_POISONED_TLP:
2608                         estr = "FUM_POISONED_TLP"; break;
2609                 case FUM_BAD_VF_QACCESS:
2610                         estr = "FUM_BAD_VF_QACCESS"; break;
2611                 case FUM_ADD_DECODE_ERR:
2612                         estr = "FUM_ADD_DECODE_ERR"; break;
2613                 case FUM_RO_ERROR:
2614                         estr = "FUM_RO_ERROR"; break;
2615                 case FUM_QPRC_CRC_ERROR:
2616                         estr = "FUM_QPRC_CRC_ERROR"; break;
2617                 case FUM_CSR_TIMEOUT:
2618                         estr = "FUM_CSR_TIMEOUT"; break;
2619                 case FUM_INVALID_TYPE:
2620                         estr = "FUM_INVALID_TYPE"; break;
2621                 case FUM_INVALID_LENGTH:
2622                         estr = "FUM_INVALID_LENGTH"; break;
2623                 case FUM_INVALID_BE:
2624                         estr = "FUM_INVALID_BE"; break;
2625                 case FUM_INVALID_ALIGN:
2626                         estr = "FUM_INVALID_ALIGN"; break;
2627                 default:
2628                         goto error;
2629                 }
2630                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2631                         estr, fault.func ? "VF" : "PF", fault.func,
2632                         fault.address, fault.specinfo);
2633         }
2634
2635         return 0;
2636 error:
2637         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2638         return err;
2639 }
2640
2641 /**
2642  * PF interrupt handler triggered by NIC for handling specific interrupt.
2643  *
2644  * @param handle
2645  *  Pointer to interrupt handle.
2646  * @param param
2647  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2648  *
2649  * @return
2650  *  void
2651  */
2652 static void
2653 fm10k_dev_interrupt_handler_pf(void *param)
2654 {
2655         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2657         uint32_t cause, status;
2658         struct fm10k_dev_info *dev_info =
2659                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2660         int status_mbx;
2661         s32 err;
2662
2663         if (hw->mac.type != fm10k_mac_pf)
2664                 return;
2665
2666         cause = FM10K_READ_REG(hw, FM10K_EICR);
2667
2668         /* Handle PCI fault cases */
2669         if (cause & FM10K_EICR_FAULT_MASK) {
2670                 PMD_INIT_LOG(ERR, "INT: find fault!");
2671                 fm10k_dev_handle_fault(hw, cause);
2672         }
2673
2674         /* Handle switch up/down */
2675         if (cause & FM10K_EICR_SWITCHNOTREADY)
2676                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2677
2678         if (cause & FM10K_EICR_SWITCHREADY) {
2679                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2680                 if (dev_info->sm_down == 1) {
2681                         fm10k_mbx_lock(hw);
2682
2683                         /* For recreating logical ports */
2684                         status_mbx = hw->mac.ops.update_lport_state(hw,
2685                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2686                         if (status_mbx == FM10K_SUCCESS)
2687                                 PMD_INIT_LOG(INFO,
2688                                         "INT: Recreated Logical port");
2689                         else
2690                                 PMD_INIT_LOG(INFO,
2691                                         "INT: Logical ports weren't recreated");
2692
2693                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2694                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2695                         if (status_mbx != FM10K_SUCCESS)
2696                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2697
2698                         fm10k_mbx_unlock(hw);
2699
2700                         /* first clear the internal SW recording structure */
2701                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2702                                                 ETH_MQ_RX_VMDQ_FLAG))
2703                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2704                                         false);
2705
2706                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2707                                         MAIN_VSI_POOL_NUMBER);
2708
2709                         /*
2710                          * Add default mac address and vlan for the logical
2711                          * ports that have been created, leave to the
2712                          * application to fully recover Rx filtering.
2713                          */
2714                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2715                                         MAIN_VSI_POOL_NUMBER);
2716
2717                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2718                                                 ETH_MQ_RX_VMDQ_FLAG))
2719                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2720                                         true);
2721
2722                         dev_info->sm_down = 0;
2723                         _rte_eth_dev_callback_process(dev,
2724                                         RTE_ETH_EVENT_INTR_LSC,
2725                                         NULL);
2726                 }
2727         }
2728
2729         /* Handle mailbox message */
2730         fm10k_mbx_lock(hw);
2731         err = hw->mbx.ops.process(hw, &hw->mbx);
2732         fm10k_mbx_unlock(hw);
2733
2734         if (err == FM10K_ERR_RESET_REQUESTED) {
2735                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2736                 dev_info->sm_down = 1;
2737                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2738                                 NULL);
2739         }
2740
2741         /* Handle SRAM error */
2742         if (cause & FM10K_EICR_SRAMERROR) {
2743                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2744
2745                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2746                 /* Write to clear pending bits */
2747                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2748
2749                 /* Todo: print out error message after shared code  updates */
2750         }
2751
2752         /* Clear these 3 events if having any */
2753         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2754                  FM10K_EICR_SWITCHREADY;
2755         if (cause)
2756                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2757
2758         /* Re-enable interrupt from device side */
2759         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2760                                         FM10K_ITR_MASK_CLEAR);
2761         /* Re-enable interrupt from host side */
2762         rte_intr_enable(dev->intr_handle);
2763 }
2764
2765 /**
2766  * VF interrupt handler triggered by NIC for handling specific interrupt.
2767  *
2768  * @param handle
2769  *  Pointer to interrupt handle.
2770  * @param param
2771  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2772  *
2773  * @return
2774  *  void
2775  */
2776 static void
2777 fm10k_dev_interrupt_handler_vf(void *param)
2778 {
2779         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2780         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2781         struct fm10k_mbx_info *mbx = &hw->mbx;
2782         struct fm10k_dev_info *dev_info =
2783                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2784         const enum fm10k_mbx_state state = mbx->state;
2785         int status_mbx;
2786
2787         if (hw->mac.type != fm10k_mac_vf)
2788                 return;
2789
2790         /* Handle mailbox message if lock is acquired */
2791         fm10k_mbx_lock(hw);
2792         hw->mbx.ops.process(hw, &hw->mbx);
2793         fm10k_mbx_unlock(hw);
2794
2795         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2796                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2797
2798                 fm10k_mbx_lock(hw);
2799                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2800                                 MAX_LPORT_NUM, 1);
2801                 fm10k_mbx_unlock(hw);
2802
2803                 /* Setting reset flag */
2804                 dev_info->sm_down = 1;
2805                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2806                                 NULL);
2807         }
2808
2809         if (dev_info->sm_down == 1 &&
2810                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2811                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2812                 fm10k_mbx_lock(hw);
2813                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2814                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2815                 if (status_mbx != FM10K_SUCCESS)
2816                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2817                 fm10k_mbx_unlock(hw);
2818
2819                 /* first clear the internal SW recording structure */
2820                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2821                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2822                                 MAIN_VSI_POOL_NUMBER);
2823
2824                 /*
2825                  * Add default mac address and vlan for the logical ports that
2826                  * have been created, leave to the application to fully recover
2827                  * Rx filtering.
2828                  */
2829                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2830                                 MAIN_VSI_POOL_NUMBER);
2831                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2832
2833                 dev_info->sm_down = 0;
2834                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2835                                 NULL);
2836         }
2837
2838         /* Re-enable interrupt from device side */
2839         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2840                                         FM10K_ITR_MASK_CLEAR);
2841         /* Re-enable interrupt from host side */
2842         rte_intr_enable(dev->intr_handle);
2843 }
2844
2845 /* Mailbox message handler in VF */
2846 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2847         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2848         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2849         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2850         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2851 };
2852
2853 static int
2854 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2855 {
2856         int err = 0;
2857
2858         /* Initialize mailbox lock */
2859         fm10k_mbx_initlock(hw);
2860
2861         /* Replace default message handler with new ones */
2862         if (hw->mac.type == fm10k_mac_vf)
2863                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2864
2865         if (err) {
2866                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2867                                 err);
2868                 return err;
2869         }
2870         /* Connect to SM for PF device or PF for VF device */
2871         return hw->mbx.ops.connect(hw, &hw->mbx);
2872 }
2873
2874 static void
2875 fm10k_close_mbx_service(struct fm10k_hw *hw)
2876 {
2877         /* Disconnect from SM for PF device or PF for VF device */
2878         hw->mbx.ops.disconnect(hw, &hw->mbx);
2879 }
2880
2881 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2882         .dev_configure          = fm10k_dev_configure,
2883         .dev_start              = fm10k_dev_start,
2884         .dev_stop               = fm10k_dev_stop,
2885         .dev_close              = fm10k_dev_close,
2886         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2887         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2888         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2889         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2890         .stats_get              = fm10k_stats_get,
2891         .xstats_get             = fm10k_xstats_get,
2892         .xstats_get_names       = fm10k_xstats_get_names,
2893         .stats_reset            = fm10k_stats_reset,
2894         .xstats_reset           = fm10k_stats_reset,
2895         .link_update            = fm10k_link_update,
2896         .dev_infos_get          = fm10k_dev_infos_get,
2897         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2898         .vlan_filter_set        = fm10k_vlan_filter_set,
2899         .vlan_offload_set       = fm10k_vlan_offload_set,
2900         .mac_addr_add           = fm10k_macaddr_add,
2901         .mac_addr_remove        = fm10k_macaddr_remove,
2902         .rx_queue_start         = fm10k_dev_rx_queue_start,
2903         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2904         .tx_queue_start         = fm10k_dev_tx_queue_start,
2905         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2906         .rx_queue_setup         = fm10k_rx_queue_setup,
2907         .rx_queue_release       = fm10k_rx_queue_release,
2908         .tx_queue_setup         = fm10k_tx_queue_setup,
2909         .tx_queue_release       = fm10k_tx_queue_release,
2910         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2911         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2912         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2913         .reta_update            = fm10k_reta_update,
2914         .reta_query             = fm10k_reta_query,
2915         .rss_hash_update        = fm10k_rss_hash_update,
2916         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2917 };
2918
2919 static int ftag_check_handler(__rte_unused const char *key,
2920                 const char *value, __rte_unused void *opaque)
2921 {
2922         if (strcmp(value, "1"))
2923                 return -1;
2924
2925         return 0;
2926 }
2927
2928 static int
2929 fm10k_check_ftag(struct rte_devargs *devargs)
2930 {
2931         struct rte_kvargs *kvlist;
2932         const char *ftag_key = "enable_ftag";
2933
2934         if (devargs == NULL)
2935                 return 0;
2936
2937         kvlist = rte_kvargs_parse(devargs->args, NULL);
2938         if (kvlist == NULL)
2939                 return 0;
2940
2941         if (!rte_kvargs_count(kvlist, ftag_key)) {
2942                 rte_kvargs_free(kvlist);
2943                 return 0;
2944         }
2945         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2946         if (rte_kvargs_process(kvlist, ftag_key,
2947                                 ftag_check_handler, NULL) < 0) {
2948                 rte_kvargs_free(kvlist);
2949                 return 0;
2950         }
2951         rte_kvargs_free(kvlist);
2952
2953         return 1;
2954 }
2955
2956 static uint16_t
2957 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2958                     uint16_t nb_pkts)
2959 {
2960         uint16_t nb_tx = 0;
2961         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2962
2963         while (nb_pkts) {
2964                 uint16_t ret, num;
2965
2966                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2967                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2968                                                  num);
2969                 nb_tx += ret;
2970                 nb_pkts -= ret;
2971                 if (ret < num)
2972                         break;
2973         }
2974
2975         return nb_tx;
2976 }
2977
2978 static void __attribute__((cold))
2979 fm10k_set_tx_function(struct rte_eth_dev *dev)
2980 {
2981         struct fm10k_tx_queue *txq;
2982         int i;
2983         int use_sse = 1;
2984         uint16_t tx_ftag_en = 0;
2985
2986         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2987                 /* primary process has set the ftag flag and txq_flags */
2988                 txq = dev->data->tx_queues[0];
2989                 if (fm10k_tx_vec_condition_check(txq)) {
2990                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2991                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2992                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2993                 } else {
2994                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2995                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2996                         dev->tx_pkt_prepare = NULL;
2997                 }
2998                 return;
2999         }
3000
3001         if (fm10k_check_ftag(dev->device->devargs))
3002                 tx_ftag_en = 1;
3003
3004         for (i = 0; i < dev->data->nb_tx_queues; i++) {
3005                 txq = dev->data->tx_queues[i];
3006                 txq->tx_ftag_en = tx_ftag_en;
3007                 /* Check if Vector Tx is satisfied */
3008                 if (fm10k_tx_vec_condition_check(txq))
3009                         use_sse = 0;
3010         }
3011
3012         if (use_sse) {
3013                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
3014                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3015                         txq = dev->data->tx_queues[i];
3016                         fm10k_txq_vec_setup(txq);
3017                 }
3018                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
3019                 dev->tx_pkt_prepare = NULL;
3020         } else {
3021                 dev->tx_pkt_burst = fm10k_xmit_pkts;
3022                 dev->tx_pkt_prepare = fm10k_prep_pkts;
3023                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
3024         }
3025 }
3026
3027 static void __attribute__((cold))
3028 fm10k_set_rx_function(struct rte_eth_dev *dev)
3029 {
3030         struct fm10k_dev_info *dev_info =
3031                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3032         uint16_t i, rx_using_sse;
3033         uint16_t rx_ftag_en = 0;
3034
3035         if (fm10k_check_ftag(dev->device->devargs))
3036                 rx_ftag_en = 1;
3037
3038         /* In order to allow Vector Rx there are a few configuration
3039          * conditions to be met.
3040          */
3041         if (!fm10k_rx_vec_condition_check(dev) &&
3042                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3043                 if (dev->data->scattered_rx)
3044                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3045                 else
3046                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3047         } else if (dev->data->scattered_rx)
3048                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3049         else
3050                 dev->rx_pkt_burst = fm10k_recv_pkts;
3051
3052         rx_using_sse =
3053                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3054                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3055
3056         if (rx_using_sse)
3057                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3058         else
3059                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3060
3061         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3062                 return;
3063
3064         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3065                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3066
3067                 rxq->rx_using_sse = rx_using_sse;
3068                 rxq->rx_ftag_en = rx_ftag_en;
3069         }
3070 }
3071
3072 static void
3073 fm10k_params_init(struct rte_eth_dev *dev)
3074 {
3075         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3076         struct fm10k_dev_info *info =
3077                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3078
3079         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3080          * there is no way to get link status without reading BAR4.  Until this
3081          * works, assume we have maximum bandwidth.
3082          * @todo - fix bus info
3083          */
3084         hw->bus_caps.speed = fm10k_bus_speed_8000;
3085         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3086         hw->bus_caps.payload = fm10k_bus_payload_512;
3087         hw->bus.speed = fm10k_bus_speed_8000;
3088         hw->bus.width = fm10k_bus_width_pcie_x8;
3089         hw->bus.payload = fm10k_bus_payload_256;
3090
3091         info->rx_vec_allowed = true;
3092 }
3093
3094 static int
3095 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3096 {
3097         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3098         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3099         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3100         int diag, i;
3101         struct fm10k_macvlan_filter_info *macvlan;
3102
3103         PMD_INIT_FUNC_TRACE();
3104
3105         dev->dev_ops = &fm10k_eth_dev_ops;
3106         dev->rx_pkt_burst = &fm10k_recv_pkts;
3107         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3108         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3109
3110         /*
3111          * Primary process does the whole initialization, for secondary
3112          * processes, we just select the same Rx and Tx function as primary.
3113          */
3114         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3115                 fm10k_set_rx_function(dev);
3116                 fm10k_set_tx_function(dev);
3117                 return 0;
3118         }
3119
3120         rte_eth_copy_pci_info(dev, pdev);
3121
3122         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3123         memset(macvlan, 0, sizeof(*macvlan));
3124         /* Vendor and Device ID need to be set before init of shared code */
3125         memset(hw, 0, sizeof(*hw));
3126         hw->device_id = pdev->id.device_id;
3127         hw->vendor_id = pdev->id.vendor_id;
3128         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3129         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3130         hw->revision_id = 0;
3131         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3132         if (hw->hw_addr == NULL) {
3133                 PMD_INIT_LOG(ERR, "Bad mem resource."
3134                         " Try to blacklist unused devices.");
3135                 return -EIO;
3136         }
3137
3138         /* Store fm10k_adapter pointer */
3139         hw->back = dev->data->dev_private;
3140
3141         /* Initialize the shared code */
3142         diag = fm10k_init_shared_code(hw);
3143         if (diag != FM10K_SUCCESS) {
3144                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3145                 return -EIO;
3146         }
3147
3148         /* Initialize parameters */
3149         fm10k_params_init(dev);
3150
3151         /* Initialize the hw */
3152         diag = fm10k_init_hw(hw);
3153         if (diag != FM10K_SUCCESS) {
3154                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3155                 return -EIO;
3156         }
3157
3158         /* Initialize MAC address(es) */
3159         dev->data->mac_addrs = rte_zmalloc("fm10k",
3160                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3161         if (dev->data->mac_addrs == NULL) {
3162                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3163                 return -ENOMEM;
3164         }
3165
3166         diag = fm10k_read_mac_addr(hw);
3167
3168         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3169                         &dev->data->mac_addrs[0]);
3170
3171         if (diag != FM10K_SUCCESS ||
3172                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3173
3174                 /* Generate a random addr */
3175                 eth_random_addr(hw->mac.addr);
3176                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3177                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3178                 &dev->data->mac_addrs[0]);
3179         }
3180
3181         /* Reset the hw statistics */
3182         fm10k_stats_reset(dev);
3183
3184         /* Reset the hw */
3185         diag = fm10k_reset_hw(hw);
3186         if (diag != FM10K_SUCCESS) {
3187                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3188                 return -EIO;
3189         }
3190
3191         /* Setup mailbox service */
3192         diag = fm10k_setup_mbx_service(hw);
3193         if (diag != FM10K_SUCCESS) {
3194                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3195                 return -EIO;
3196         }
3197
3198         /*PF/VF has different interrupt handling mechanism */
3199         if (hw->mac.type == fm10k_mac_pf) {
3200                 /* register callback func to eal lib */
3201                 rte_intr_callback_register(intr_handle,
3202                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3203
3204                 /* enable MISC interrupt */
3205                 fm10k_dev_enable_intr_pf(dev);
3206         } else { /* VF */
3207                 rte_intr_callback_register(intr_handle,
3208                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3209
3210                 fm10k_dev_enable_intr_vf(dev);
3211         }
3212
3213         /* Enable intr after callback registered */
3214         rte_intr_enable(intr_handle);
3215
3216         hw->mac.ops.update_int_moderator(hw);
3217
3218         /* Make sure Switch Manager is ready before going forward. */
3219         if (hw->mac.type == fm10k_mac_pf) {
3220                 int switch_ready = 0;
3221
3222                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3223                         fm10k_mbx_lock(hw);
3224                         hw->mac.ops.get_host_state(hw, &switch_ready);
3225                         fm10k_mbx_unlock(hw);
3226                         if (switch_ready)
3227                                 break;
3228                         /* Delay some time to acquire async LPORT_MAP info. */
3229                         rte_delay_us(WAIT_SWITCH_MSG_US);
3230                 }
3231
3232                 if (switch_ready == 0) {
3233                         PMD_INIT_LOG(ERR, "switch is not ready");
3234                         return -1;
3235                 }
3236         }
3237
3238         /*
3239          * Below function will trigger operations on mailbox, acquire lock to
3240          * avoid race condition from interrupt handler. Operations on mailbox
3241          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3242          * will handle and generate an interrupt to our side. Then,  FIFO in
3243          * mailbox will be touched.
3244          */
3245         fm10k_mbx_lock(hw);
3246         /* Enable port first */
3247         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3248                                         MAX_LPORT_NUM, 1);
3249
3250         /* Set unicast mode by default. App can change to other mode in other
3251          * API func.
3252          */
3253         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3254                                         FM10K_XCAST_MODE_NONE);
3255
3256         fm10k_mbx_unlock(hw);
3257
3258         /* Make sure default VID is ready before going forward. */
3259         if (hw->mac.type == fm10k_mac_pf) {
3260                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3261                         if (hw->mac.default_vid)
3262                                 break;
3263                         /* Delay some time to acquire async port VLAN info. */
3264                         rte_delay_us(WAIT_SWITCH_MSG_US);
3265                 }
3266
3267                 if (!hw->mac.default_vid) {
3268                         PMD_INIT_LOG(ERR, "default VID is not ready");
3269                         return -1;
3270                 }
3271         }
3272
3273         /* Add default mac address */
3274         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3275                 MAIN_VSI_POOL_NUMBER);
3276
3277         return 0;
3278 }
3279
3280 static int
3281 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3282 {
3283         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3284         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3285         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3286         PMD_INIT_FUNC_TRACE();
3287
3288         /* only uninitialize in the primary process */
3289         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3290                 return 0;
3291
3292         /* safe to close dev here */
3293         fm10k_dev_close(dev);
3294
3295         dev->dev_ops = NULL;
3296         dev->rx_pkt_burst = NULL;
3297         dev->tx_pkt_burst = NULL;
3298
3299         /* disable uio/vfio intr */
3300         rte_intr_disable(intr_handle);
3301
3302         /*PF/VF has different interrupt handling mechanism */
3303         if (hw->mac.type == fm10k_mac_pf) {
3304                 /* disable interrupt */
3305                 fm10k_dev_disable_intr_pf(dev);
3306
3307                 /* unregister callback func to eal lib */
3308                 rte_intr_callback_unregister(intr_handle,
3309                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3310         } else {
3311                 /* disable interrupt */
3312                 fm10k_dev_disable_intr_vf(dev);
3313
3314                 rte_intr_callback_unregister(intr_handle,
3315                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3316         }
3317
3318         /* free mac memory */
3319         if (dev->data->mac_addrs) {
3320                 rte_free(dev->data->mac_addrs);
3321                 dev->data->mac_addrs = NULL;
3322         }
3323
3324         memset(hw, 0, sizeof(*hw));
3325
3326         return 0;
3327 }
3328
3329 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3330         struct rte_pci_device *pci_dev)
3331 {
3332         return rte_eth_dev_pci_generic_probe(pci_dev,
3333                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3334 }
3335
3336 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3337 {
3338         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3339 }
3340
3341 /*
3342  * The set of PCI devices this driver supports. This driver will enable both PF
3343  * and SRIOV-VF devices.
3344  */
3345 static const struct rte_pci_id pci_id_fm10k_map[] = {
3346         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3347         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3348         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3349         { .vendor_id = 0, /* sentinel */ },
3350 };
3351
3352 static struct rte_pci_driver rte_pmd_fm10k = {
3353         .id_table = pci_id_fm10k_map,
3354         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3355                      RTE_PCI_DRV_IOVA_AS_VA,
3356         .probe = eth_fm10k_pci_probe,
3357         .remove = eth_fm10k_pci_remove,
3358 };
3359
3360 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3361 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3362 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3363
3364 RTE_INIT(fm10k_init_log);
3365 static void
3366 fm10k_init_log(void)
3367 {
3368         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3369         if (fm10k_logtype_init >= 0)
3370                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3371         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3372         if (fm10k_logtype_driver >= 0)
3373                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3374 }