drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451         struct rte_eth_dev_info dev_info;
 452         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
 453         uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
 454
 455         PMD_INIT_FUNC_TRACE();
 456
 457         if ((rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP) == 0)
 458                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 459
 460         fm10k_dev_infos_get(dev, &dev_info);
 461         if ((rx_offloads & dev_info.rx_offload_capa) != rx_offloads) {
 462                 PMD_DRV_LOG(ERR, "Some Rx offloads are not supported "
 463                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 464                             rx_offloads, dev_info.rx_offload_capa);
 465                 return -ENOTSUP;
 466         }
 467         if ((tx_offloads & dev_info.tx_offload_capa) != tx_offloads) {
 468                 PMD_DRV_LOG(ERR, "Some Tx offloads are not supported "
 469                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 470                             tx_offloads, dev_info.tx_offload_capa);
 471                 return -ENOTSUP;
 472         }
 473
 474         /* multipe queue mode checking */
 475         ret  = fm10k_check_mq_mode(dev);
 476         if (ret != 0) {
 477                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 478                             ret);
 479                 return ret;
 480         }
 481
 482         dev->data->scattered_rx = 0;
 483
 484         return 0;
 485 }
 486
 487 /* fls = find last set bit = 32 minus the number of leading zeros */
 488 #ifndef fls
 489 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 490 #endif
 491
 492 static void
 493 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 494 {
 495         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 496         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 497         uint32_t i;
 498
 499         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 500
 501         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 502                 if (!vmdq_conf->pool_map[i].pools)
 503                         continue;
 504                 fm10k_mbx_lock(hw);
 505                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 506                 fm10k_mbx_unlock(hw);
 507         }
 508 }
 509
 510 static void
 511 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 512 {
 513         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 514
 515         /* Add default mac address */
 516         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 517                 MAIN_VSI_POOL_NUMBER);
 518 }
 519
 520 static void
 521 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 522 {
 523         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 524         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 525         uint32_t mrqc, *key, i, reta, j;
 526         uint64_t hf;
 527
 528 #define RSS_KEY_SIZE 40
 529         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 530                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 531                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 532                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 533                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 534                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 535         };
 536
 537         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 538                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 539                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 540                 return;
 541         }
 542
 543         /* random key is rss_intel_key (default) or user provided (rss_key) */
 544         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 545                 key = (uint32_t *)rss_intel_key;
 546         else
 547                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 548
 549         /* Now fill our hash function seeds, 4 bytes at a time */
 550         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 551                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 552
 553         /*
 554          * Fill in redirection table
 555          * The byte-swap is needed because NIC registers are in
 556          * little-endian order.
 557          */
 558         reta = 0;
 559         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 560                 if (j == dev->data->nb_rx_queues)
 561                         j = 0;
 562                 reta = (reta << CHAR_BIT) | j;
 563                 if ((i & 3) == 3)
 564                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 565                                         rte_bswap32(reta));
 566         }
 567
 568         /*
 569          * Generate RSS hash based on packet types, TCP/UDP
 570          * port numbers and/or IPv4/v6 src and dst addresses
 571          */
 572         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 573         mrqc = 0;
 574         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 578         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 581         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 582         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 583
 584         if (mrqc == 0) {
 585                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 586                         "supported", hf);
 587                 return;
 588         }
 589
 590         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 591 }
 592
 593 static void
 594 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 595 {
 596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 597         uint32_t i;
 598
 599         for (i = 0; i < nb_lport_new; i++) {
 600                 /* Set unicast mode by default. App can change
 601                  * to other mode in other API func.
 602                  */
 603                 fm10k_mbx_lock(hw);
 604                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 605                         FM10K_XCAST_MODE_NONE);
 606                 fm10k_mbx_unlock(hw);
 607         }
 608 }
 609
 610 static void
 611 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 612 {
 613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 615         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 616         struct fm10k_macvlan_filter_info *macvlan;
 617         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 618         uint16_t nb_lport_new;
 619
 620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 621         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 622
 623         fm10k_dev_rss_configure(dev);
 624
 625         /* only PF supports VMDQ */
 626         if (hw->mac.type != fm10k_mac_pf)
 627                 return;
 628
 629         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 630                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 631
 632         /* no pool number change, no need to update logic port and VLAN/MAC */
 633         if (macvlan->nb_queue_pools == nb_queue_pools)
 634                 return;
 635
 636         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 637         fm10k_dev_logic_port_update(dev, nb_lport_new);
 638
 639         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 640         memset(dev->data->mac_addrs, 0,
 641                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 642         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 643                 &dev->data->mac_addrs[0]);
 644         memset(macvlan, 0, sizeof(*macvlan));
 645         macvlan->nb_queue_pools = nb_queue_pools;
 646
 647         if (nb_queue_pools)
 648                 fm10k_dev_vmdq_rx_configure(dev);
 649         else
 650                 fm10k_dev_pf_main_vsi_reset(dev);
 651 }
 652
 653 static int
 654 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 655 {
 656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 657         int i, ret;
 658         struct fm10k_tx_queue *txq;
 659         uint64_t base_addr;
 660         uint32_t size;
 661
 662         /* Disable TXINT to avoid possible interrupt */
 663         for (i = 0; i < hw->mac.max_queues; i++)
 664                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 665                                 3 << FM10K_TXINT_TIMER_SHIFT);
 666
 667         /* Setup TX queue */
 668         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 669                 txq = dev->data->tx_queues[i];
 670                 base_addr = txq->hw_ring_phys_addr;
 671                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 672
 673                 /* disable queue to avoid issues while updating state */
 674                 ret = tx_queue_disable(hw, i);
 675                 if (ret) {
 676                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 677                         return -1;
 678                 }
 679                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 680                  * register is read-only for VF.
 681                  */
 682                 if (fm10k_check_ftag(dev->device->devargs)) {
 683                         if (hw->mac.type == fm10k_mac_pf) {
 684                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 685                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 686                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 687                         } else {
 688                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 689                                 return -ENOTSUP;
 690                         }
 691                 }
 692
 693                 /* set location and size for descriptor ring */
 694                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 695                                 base_addr & UINT64_LOWER_32BITS_MASK);
 696                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 697                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 698                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 699
 700                 /* assign default SGLORT for each TX queue by PF */
 701                 if (hw->mac.type == fm10k_mac_pf)
 702                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 703         }
 704
 705         /* set up vector or scalar TX function as appropriate */
 706         fm10k_set_tx_function(dev);
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         struct fm10k_macvlan_filter_info *macvlan;
 716         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 717         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 718         int i, ret;
 719         struct fm10k_rx_queue *rxq;
 720         uint64_t base_addr;
 721         uint32_t size;
 722         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 723         uint32_t logic_port = hw->mac.dglort_map;
 724         uint16_t buf_size;
 725         uint16_t queue_stride = 0;
 726
 727         /* enable RXINT for interrupt mode */
 728         i = 0;
 729         if (rte_intr_dp_is_en(intr_handle)) {
 730                 for (; i < dev->data->nb_rx_queues; i++) {
 731                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 732                         if (hw->mac.type == fm10k_mac_pf)
 733                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 734                                         FM10K_ITR_AUTOMASK |
 735                                         FM10K_ITR_MASK_CLEAR);
 736                         else
 737                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 738                                         FM10K_ITR_AUTOMASK |
 739                                         FM10K_ITR_MASK_CLEAR);
 740                 }
 741         }
 742         /* Disable other RXINT to avoid possible interrupt */
 743         for (; i < hw->mac.max_queues; i++)
 744                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 745                         3 << FM10K_RXINT_TIMER_SHIFT);
 746
 747         /* Setup RX queues */
 748         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 749                 rxq = dev->data->rx_queues[i];
 750                 base_addr = rxq->hw_ring_phys_addr;
 751                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 752
 753                 /* disable queue to avoid issues while updating state */
 754                 ret = rx_queue_disable(hw, i);
 755                 if (ret) {
 756                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 757                         return -1;
 758                 }
 759
 760                 /* Setup the Base and Length of the Rx Descriptor Ring */
 761                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 762                                 base_addr & UINT64_LOWER_32BITS_MASK);
 763                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 764                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 765                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 766
 767                 /* Configure the Rx buffer size for one buff without split */
 768                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 769                         RTE_PKTMBUF_HEADROOM);
 770                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 771                  * reserved for this purpose, and the worst case could be 511B.
 772                  * But SRR reg assumes all buffers have the same size. In order
 773                  * to fill the gap, we'll have to consider the worst case and
 774                  * assume 512B is reserved. If we don't do so, it's possible
 775                  * for HW to overwrite data to next mbuf.
 776                  */
 777                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 778
 779                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 780                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 781                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 782
 783                 /* It adds dual VLAN length for supporting dual VLAN */
 784                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 785                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 786                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 787                         uint32_t reg;
 788                         dev->data->scattered_rx = 1;
 789                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 790                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 791                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 792                 }
 793
 794                 /* Enable drop on empty, it's RO for VF */
 795                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 796                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 797
 798                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 799                 FM10K_WRITE_FLUSH(hw);
 800         }
 801
 802         /* Configure VMDQ/RSS if applicable */
 803         fm10k_dev_mq_rx_configure(dev);
 804
 805         /* Decide the best RX function */
 806         fm10k_set_rx_function(dev);
 807
 808         /* update RX_SGLORT for loopback suppress*/
 809         if (hw->mac.type != fm10k_mac_pf)
 810                 return 0;
 811         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 812         if (macvlan->nb_queue_pools)
 813                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 814         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 815                 if (i && queue_stride && !(i % queue_stride))
 816                         logic_port++;
 817                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 818         }
 819
 820         return 0;
 821 }
 822
 823 static int
 824 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 825 {
 826         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 827         int err = -1;
 828         uint32_t reg;
 829         struct fm10k_rx_queue *rxq;
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         if (rx_queue_id < dev->data->nb_rx_queues) {
 834                 rxq = dev->data->rx_queues[rx_queue_id];
 835                 err = rx_queue_reset(rxq);
 836                 if (err == -ENOMEM) {
 837                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 838                         return err;
 839                 } else if (err == -EINVAL) {
 840                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 841                                 " %d", err);
 842                         return err;
 843                 }
 844
 845                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 846                  * Note: this must be done AFTER the queue is enabled on real
 847                  * hardware, but BEFORE the queue is enabled when using the
 848                  * emulation platform. Do it in both places for now and remove
 849                  * this comment and the following two register writes when the
 850                  * emulation platform is no longer being used.
 851                  */
 852                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854
 855                 /* Set PF ownership flag for PF devices */
 856                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 857                 if (hw->mac.type == fm10k_mac_pf)
 858                         reg |= FM10K_RXQCTL_PF;
 859                 reg |= FM10K_RXQCTL_ENABLE;
 860                 /* enable RX queue */
 861                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 862                 FM10K_WRITE_FLUSH(hw);
 863
 864                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 865                  * Note: this must be done AFTER the queue is enabled
 866                  */
 867                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 868                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 869                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 870         }
 871
 872         return err;
 873 }
 874
 875 static int
 876 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (rx_queue_id < dev->data->nb_rx_queues) {
 883                 /* Disable RX queue */
 884                 rx_queue_disable(hw, rx_queue_id);
 885
 886                 /* Free mbuf and clean HW ring */
 887                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 888                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 896 {
 897         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         /** @todo - this should be defined in the shared code */
 899 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 900         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 901         int err = 0;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (tx_queue_id < dev->data->nb_tx_queues) {
 906                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 907
 908                 q->ops->reset(q);
 909
 910                 /* reset head and tail pointers */
 911                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 912                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 913
 914                 /* enable TX queue */
 915                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 916                                         FM10K_TXDCTL_ENABLE | txdctl);
 917                 FM10K_WRITE_FLUSH(hw);
 918                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 919         } else
 920                 err = -1;
 921
 922         return err;
 923 }
 924
 925 static int
 926 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 927 {
 928         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (tx_queue_id < dev->data->nb_tx_queues) {
 933                 tx_queue_disable(hw, tx_queue_id);
 934                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 935                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 936         }
 937
 938         return 0;
 939 }
 940
 941 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 942 {
 943         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 944                 != FM10K_DGLORTMAP_NONE);
 945 }
 946
 947 static void
 948 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 949 {
 950         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return;
 958
 959         fm10k_mbx_lock(hw);
 960         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 961                                 FM10K_XCAST_MODE_PROMISC);
 962         fm10k_mbx_unlock(hw);
 963
 964         if (status != FM10K_SUCCESS)
 965                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 966 }
 967
 968 static void
 969 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint8_t mode;
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return;
 980
 981         if (dev->data->all_multicast == 1)
 982                 mode = FM10K_XCAST_MODE_ALLMULTI;
 983         else
 984                 mode = FM10K_XCAST_MODE_NONE;
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 mode);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         /* If promiscuous mode is enabled, it doesn't make sense to enable
1008          * allmulticast and disable promiscuous since fm10k only can select
1009          * one of the modes.
1010          */
1011         if (dev->data->promiscuous) {
1012                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1013                         "needn't enable allmulticast");
1014                 return;
1015         }
1016
1017         fm10k_mbx_lock(hw);
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_ALLMULTI);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         int status;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Return if it didn't acquire valid glort range */
1035         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1036                 return;
1037
1038         if (dev->data->promiscuous) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1040                         "since promisc mode is enabled");
1041                 return;
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         /* Change mode to unicast mode */
1046         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1047                                 FM10K_XCAST_MODE_NONE);
1048         fm10k_mbx_unlock(hw);
1049
1050         if (status != FM10K_SUCCESS)
1051                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1052 }
1053
1054 static void
1055 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1056 {
1057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1059         uint16_t nb_queue_pools;
1060         struct fm10k_macvlan_filter_info *macvlan;
1061
1062         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1063         nb_queue_pools = macvlan->nb_queue_pools;
1064         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1065         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1066
1067         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1068         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1069         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         hw->mac.dglort_map;
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1072         /* Configure VMDQ/RSS DGlort Decoder */
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1074
1075         /* Flow Director configurations, only queue number is valid. */
1076         dglortdec = fls(dev->data->nb_rx_queues - 1);
1077         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1078                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1080         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1081
1082         /* Invalidate all other GLORT entries */
1083         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1084                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1085                                 FM10K_DGLORTMAP_NONE);
1086 }
1087
1088 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1089 static int
1090 fm10k_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         int i, diag;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* stop, init, then start the hw */
1098         diag = fm10k_stop_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_init_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_start_hw(hw);
1111         if (diag != FM10K_SUCCESS) {
1112                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1113                 return -EIO;
1114         }
1115
1116         diag = fm10k_dev_tx_init(dev);
1117         if (diag) {
1118                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1119                 return diag;
1120         }
1121
1122         if (fm10k_dev_rxq_interrupt_setup(dev))
1123                 return -EIO;
1124
1125         diag = fm10k_dev_rx_init(dev);
1126         if (diag) {
1127                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1128                 return diag;
1129         }
1130
1131         if (hw->mac.type == fm10k_mac_pf)
1132                 fm10k_dev_dglort_map_configure(dev);
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 struct fm10k_rx_queue *rxq;
1136                 rxq = dev->data->rx_queues[i];
1137
1138                 if (rxq->rx_deferred_start)
1139                         continue;
1140                 diag = fm10k_dev_rx_queue_start(dev, i);
1141                 if (diag != 0) {
1142                         int j;
1143                         for (j = 0; j < i; ++j)
1144                                 rx_queue_clean(dev->data->rx_queues[j]);
1145                         return diag;
1146                 }
1147         }
1148
1149         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1150                 struct fm10k_tx_queue *txq;
1151                 txq = dev->data->tx_queues[i];
1152
1153                 if (txq->tx_deferred_start)
1154                         continue;
1155                 diag = fm10k_dev_tx_queue_start(dev, i);
1156                 if (diag != 0) {
1157                         int j;
1158                         for (j = 0; j < i; ++j)
1159                                 tx_queue_clean(dev->data->tx_queues[j]);
1160                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1161                                 rx_queue_clean(dev->data->rx_queues[j]);
1162                         return diag;
1163                 }
1164         }
1165
1166         /* Update default vlan when not in VMDQ mode */
1167         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1168                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1169
1170         fm10k_link_update(dev, 0);
1171
1172         return 0;
1173 }
1174
1175 static void
1176 fm10k_dev_stop(struct rte_eth_dev *dev)
1177 {
1178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1179         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1180         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1181         int i;
1182
1183         PMD_INIT_FUNC_TRACE();
1184
1185         if (dev->data->tx_queues)
1186                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1187                         fm10k_dev_tx_queue_stop(dev, i);
1188
1189         if (dev->data->rx_queues)
1190                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1191                         fm10k_dev_rx_queue_stop(dev, i);
1192
1193         /* Disable datapath event */
1194         if (rte_intr_dp_is_en(intr_handle)) {
1195                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1196                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1197                                 3 << FM10K_RXINT_TIMER_SHIFT);
1198                         if (hw->mac.type == fm10k_mac_pf)
1199                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                         else
1202                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1203                                         FM10K_ITR_MASK_SET);
1204                 }
1205         }
1206         /* Clean datapath event and queue/vec mapping */
1207         rte_intr_efd_disable(intr_handle);
1208         rte_free(intr_handle->intr_vec);
1209         intr_handle->intr_vec = NULL;
1210 }
1211
1212 static void
1213 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1214 {
1215         int i;
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         if (dev->data->tx_queues) {
1220                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1222
1223                         tx_queue_free(txq);
1224                 }
1225         }
1226
1227         if (dev->data->rx_queues) {
1228                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1229                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1230         }
1231 }
1232
1233 static void
1234 fm10k_dev_close(struct rte_eth_dev *dev)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237
1238         PMD_INIT_FUNC_TRACE();
1239
1240         fm10k_mbx_lock(hw);
1241         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1242                 MAX_LPORT_NUM, false);
1243         fm10k_mbx_unlock(hw);
1244
1245         /* allow 100ms for device to quiesce */
1246         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1247
1248         /* Stop mailbox service first */
1249         fm10k_close_mbx_service(hw);
1250         fm10k_dev_stop(dev);
1251         fm10k_dev_queue_release(dev);
1252         fm10k_stop_hw(hw);
1253 }
1254
1255 static int
1256 fm10k_link_update(struct rte_eth_dev *dev,
1257         __rte_unused int wait_to_complete)
1258 {
1259         struct fm10k_dev_info *dev_info =
1260                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1261         PMD_INIT_FUNC_TRACE();
1262
1263         /* The speed is ~50Gbps per Gen3 x8 PCIe interface. For now, we
1264          * leave the speed undefined since there is no 50Gbps Ethernet.
1265          */
1266         dev->data->dev_link.link_speed  = 0;
1267         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1268         dev->data->dev_link.link_status =
1269                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1270
1271         return 0;
1272 }
1273
1274 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1275         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1276 {
1277         unsigned i, q;
1278         unsigned count = 0;
1279
1280         if (xstats_names != NULL) {
1281                 /* Note: limit checked in rte_eth_xstats_names() */
1282
1283                 /* Global stats */
1284                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1285                         snprintf(xstats_names[count].name,
1286                                 sizeof(xstats_names[count].name),
1287                                 "%s", fm10k_hw_stats_strings[count].name);
1288                         count++;
1289                 }
1290
1291                 /* PF queue stats */
1292                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1293                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1294                                 snprintf(xstats_names[count].name,
1295                                         sizeof(xstats_names[count].name),
1296                                         "rx_q%u_%s", q,
1297                                         fm10k_hw_stats_rx_q_strings[i].name);
1298                                 count++;
1299                         }
1300                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1301                                 snprintf(xstats_names[count].name,
1302                                         sizeof(xstats_names[count].name),
1303                                         "tx_q%u_%s", q,
1304                                         fm10k_hw_stats_tx_q_strings[i].name);
1305                                 count++;
1306                         }
1307                 }
1308         }
1309         return FM10K_NB_XSTATS;
1310 }
1311
1312 static int
1313 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1314                  unsigned n)
1315 {
1316         struct fm10k_hw_stats *hw_stats =
1317                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1318         unsigned i, q, count = 0;
1319
1320         if (n < FM10K_NB_XSTATS)
1321                 return FM10K_NB_XSTATS;
1322
1323         /* Global stats */
1324         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1325                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1326                         fm10k_hw_stats_strings[count].offset);
1327                 xstats[count].id = count;
1328                 count++;
1329         }
1330
1331         /* PF queue stats */
1332         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1333                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1334                         xstats[count].value =
1335                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1336                                 fm10k_hw_stats_rx_q_strings[i].offset);
1337                         xstats[count].id = count;
1338                         count++;
1339                 }
1340                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1341                         xstats[count].value =
1342                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1343                                 fm10k_hw_stats_tx_q_strings[i].offset);
1344                         xstats[count].id = count;
1345                         count++;
1346                 }
1347         }
1348
1349         return FM10K_NB_XSTATS;
1350 }
1351
1352 static int
1353 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1354 {
1355         uint64_t ipackets, opackets, ibytes, obytes;
1356         struct fm10k_hw *hw =
1357                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1358         struct fm10k_hw_stats *hw_stats =
1359                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1360         int i;
1361
1362         PMD_INIT_FUNC_TRACE();
1363
1364         fm10k_update_hw_stats(hw, hw_stats);
1365
1366         ipackets = opackets = ibytes = obytes = 0;
1367         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1368                 (i < hw->mac.max_queues); ++i) {
1369                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1370                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1371                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1372                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1373                 ipackets += stats->q_ipackets[i];
1374                 opackets += stats->q_opackets[i];
1375                 ibytes   += stats->q_ibytes[i];
1376                 obytes   += stats->q_obytes[i];
1377         }
1378         stats->ipackets = ipackets;
1379         stats->opackets = opackets;
1380         stats->ibytes = ibytes;
1381         stats->obytes = obytes;
1382         return 0;
1383 }
1384
1385 static void
1386 fm10k_stats_reset(struct rte_eth_dev *dev)
1387 {
1388         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1389         struct fm10k_hw_stats *hw_stats =
1390                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1391
1392         PMD_INIT_FUNC_TRACE();
1393
1394         memset(hw_stats, 0, sizeof(*hw_stats));
1395         fm10k_rebind_hw_stats(hw, hw_stats);
1396 }
1397
1398 static void
1399 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1400         struct rte_eth_dev_info *dev_info)
1401 {
1402         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1403         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1404
1405         PMD_INIT_FUNC_TRACE();
1406
1407         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1408         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1409         dev_info->max_rx_queues      = hw->mac.max_queues;
1410         dev_info->max_tx_queues      = hw->mac.max_queues;
1411         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1412         dev_info->max_hash_mac_addrs = 0;
1413         dev_info->max_vfs            = pdev->max_vfs;
1414         dev_info->vmdq_pool_base     = 0;
1415         dev_info->vmdq_queue_base    = 0;
1416         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1417         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1418         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1419         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1420                                     dev_info->rx_queue_offload_capa;
1421         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1422         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1423                                     dev_info->tx_queue_offload_capa;
1424
1425         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1426         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1427
1428         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1429                 .rx_thresh = {
1430                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1431                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1432                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1433                 },
1434                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1435                 .rx_drop_en = 0,
1436                 .offloads = 0,
1437         };
1438
1439         dev_info->default_txconf = (struct rte_eth_txconf) {
1440                 .tx_thresh = {
1441                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1442                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1443                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1444                 },
1445                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1446                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1447                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1448                 .offloads = 0,
1449         };
1450
1451         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1452                 .nb_max = FM10K_MAX_RX_DESC,
1453                 .nb_min = FM10K_MIN_RX_DESC,
1454                 .nb_align = FM10K_MULT_RX_DESC,
1455         };
1456
1457         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1458                 .nb_max = FM10K_MAX_TX_DESC,
1459                 .nb_min = FM10K_MIN_TX_DESC,
1460                 .nb_align = FM10K_MULT_TX_DESC,
1461                 .nb_seg_max = FM10K_TX_MAX_SEG,
1462                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1463         };
1464
1465         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1466                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1467                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1468 }
1469
1470 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1471 static const uint32_t *
1472 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1473 {
1474         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1475             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1476                 static uint32_t ptypes[] = {
1477                         /* refers to rx_desc_to_ol_flags() */
1478                         RTE_PTYPE_L2_ETHER,
1479                         RTE_PTYPE_L3_IPV4,
1480                         RTE_PTYPE_L3_IPV4_EXT,
1481                         RTE_PTYPE_L3_IPV6,
1482                         RTE_PTYPE_L3_IPV6_EXT,
1483                         RTE_PTYPE_L4_TCP,
1484                         RTE_PTYPE_L4_UDP,
1485                         RTE_PTYPE_UNKNOWN
1486                 };
1487
1488                 return ptypes;
1489         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1490                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1491                 static uint32_t ptypes_vec[] = {
1492                         /* refers to fm10k_desc_to_pktype_v() */
1493                         RTE_PTYPE_L3_IPV4,
1494                         RTE_PTYPE_L3_IPV4_EXT,
1495                         RTE_PTYPE_L3_IPV6,
1496                         RTE_PTYPE_L3_IPV6_EXT,
1497                         RTE_PTYPE_L4_TCP,
1498                         RTE_PTYPE_L4_UDP,
1499                         RTE_PTYPE_TUNNEL_GENEVE,
1500                         RTE_PTYPE_TUNNEL_NVGRE,
1501                         RTE_PTYPE_TUNNEL_VXLAN,
1502                         RTE_PTYPE_TUNNEL_GRE,
1503                         RTE_PTYPE_UNKNOWN
1504                 };
1505
1506                 return ptypes_vec;
1507         }
1508
1509         return NULL;
1510 }
1511 #else
1512 static const uint32_t *
1513 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1514 {
1515         return NULL;
1516 }
1517 #endif
1518
1519 static int
1520 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1521 {
1522         s32 result;
1523         uint16_t mac_num = 0;
1524         uint32_t vid_idx, vid_bit, mac_index;
1525         struct fm10k_hw *hw;
1526         struct fm10k_macvlan_filter_info *macvlan;
1527         struct rte_eth_dev_data *data = dev->data;
1528
1529         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1530         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1531
1532         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1533                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1534                 return -EINVAL;
1535         }
1536
1537         if (vlan_id > ETH_VLAN_ID_MAX) {
1538                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1539                 return -EINVAL;
1540         }
1541
1542         vid_idx = FM10K_VFTA_IDX(vlan_id);
1543         vid_bit = FM10K_VFTA_BIT(vlan_id);
1544         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1545         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1546                 return 0;
1547         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1548         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1549                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1550                         "in the VLAN filter table");
1551                 return -EINVAL;
1552         }
1553
1554         fm10k_mbx_lock(hw);
1555         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1556         fm10k_mbx_unlock(hw);
1557         if (result != FM10K_SUCCESS) {
1558                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1559                 return -EIO;
1560         }
1561
1562         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1563                         (result == FM10K_SUCCESS); mac_index++) {
1564                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1565                         continue;
1566                 if (mac_num > macvlan->mac_num - 1) {
1567                         PMD_INIT_LOG(ERR, "MAC address number "
1568                                         "not match");
1569                         break;
1570                 }
1571                 fm10k_mbx_lock(hw);
1572                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1573                         data->mac_addrs[mac_index].addr_bytes,
1574                         vlan_id, on, 0);
1575                 fm10k_mbx_unlock(hw);
1576                 mac_num++;
1577         }
1578         if (result != FM10K_SUCCESS) {
1579                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1580                 return -EIO;
1581         }
1582
1583         if (on) {
1584                 macvlan->vlan_num++;
1585                 macvlan->vfta[vid_idx] |= vid_bit;
1586         } else {
1587                 macvlan->vlan_num--;
1588                 macvlan->vfta[vid_idx] &= ~vid_bit;
1589         }
1590         return 0;
1591 }
1592
1593 static int
1594 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1595 {
1596         if (mask & ETH_VLAN_STRIP_MASK) {
1597                 if (!(dev->data->dev_conf.rxmode.offloads &
1598                         DEV_RX_OFFLOAD_VLAN_STRIP))
1599                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1600                                         "always on in fm10k");
1601         }
1602
1603         if (mask & ETH_VLAN_EXTEND_MASK) {
1604                 if (dev->data->dev_conf.rxmode.offloads &
1605                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1606                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1607                                         "supported in fm10k");
1608         }
1609
1610         if (mask & ETH_VLAN_FILTER_MASK) {
1611                 if (!(dev->data->dev_conf.rxmode.offloads &
1612                         DEV_RX_OFFLOAD_VLAN_FILTER))
1613                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1614         }
1615
1616         return 0;
1617 }
1618
1619 /* Add/Remove a MAC address, and update filters to main VSI */
1620 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1621                 const u8 *mac, bool add, uint32_t pool)
1622 {
1623         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1624         struct fm10k_macvlan_filter_info *macvlan;
1625         uint32_t i, j, k;
1626
1627         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1628
1629         if (pool != MAIN_VSI_POOL_NUMBER) {
1630                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1631                         "mac to pool %u", pool);
1632                 return;
1633         }
1634         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1635                 if (!macvlan->vfta[j])
1636                         continue;
1637                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1638                         if (!(macvlan->vfta[j] & (1 << k)))
1639                                 continue;
1640                         if (i + 1 > macvlan->vlan_num) {
1641                                 PMD_INIT_LOG(ERR, "vlan number not match");
1642                                 return;
1643                         }
1644                         fm10k_mbx_lock(hw);
1645                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1646                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1647                         fm10k_mbx_unlock(hw);
1648                         i++;
1649                 }
1650         }
1651 }
1652
1653 /* Add/Remove a MAC address, and update filters to VMDQ */
1654 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1655                 const u8 *mac, bool add, uint32_t pool)
1656 {
1657         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1658         struct fm10k_macvlan_filter_info *macvlan;
1659         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1660         uint32_t i;
1661
1662         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1663         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1664
1665         if (pool > macvlan->nb_queue_pools) {
1666                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1667                         " Max pool is %u",
1668                         pool, macvlan->nb_queue_pools);
1669                 return;
1670         }
1671         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1672                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1673                         continue;
1674                 fm10k_mbx_lock(hw);
1675                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1676                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1677                 fm10k_mbx_unlock(hw);
1678         }
1679 }
1680
1681 /* Add/Remove a MAC address, and update filters */
1682 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1683                 const u8 *mac, bool add, uint32_t pool)
1684 {
1685         struct fm10k_macvlan_filter_info *macvlan;
1686
1687         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1688
1689         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1690                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1691         else
1692                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1693
1694         if (add)
1695                 macvlan->mac_num++;
1696         else
1697                 macvlan->mac_num--;
1698 }
1699
1700 /* Add a MAC address, and update filters */
1701 static int
1702 fm10k_macaddr_add(struct rte_eth_dev *dev,
1703                 struct ether_addr *mac_addr,
1704                 uint32_t index,
1705                 uint32_t pool)
1706 {
1707         struct fm10k_macvlan_filter_info *macvlan;
1708
1709         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1710         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1711         macvlan->mac_vmdq_id[index] = pool;
1712         return 0;
1713 }
1714
1715 /* Remove a MAC address, and update filters */
1716 static void
1717 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1718 {
1719         struct rte_eth_dev_data *data = dev->data;
1720         struct fm10k_macvlan_filter_info *macvlan;
1721
1722         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1723         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1724                         FALSE, macvlan->mac_vmdq_id[index]);
1725         macvlan->mac_vmdq_id[index] = 0;
1726 }
1727
1728 static inline int
1729 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1730 {
1731         if ((request < min) || (request > max) || ((request % mult) != 0))
1732                 return -1;
1733         else
1734                 return 0;
1735 }
1736
1737
1738 static inline int
1739 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1740 {
1741         if ((request < min) || (request > max) || ((div % request) != 0))
1742                 return -1;
1743         else
1744                 return 0;
1745 }
1746
1747 static inline int
1748 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1749 {
1750         uint16_t rx_free_thresh;
1751
1752         if (conf->rx_free_thresh == 0)
1753                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1754         else
1755                 rx_free_thresh = conf->rx_free_thresh;
1756
1757         /* make sure the requested threshold satisfies the constraints */
1758         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1759                         FM10K_RX_FREE_THRESH_MAX(q),
1760                         FM10K_RX_FREE_THRESH_DIV(q),
1761                         rx_free_thresh)) {
1762                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1763                         "less than or equal to %u, "
1764                         "greater than or equal to %u, "
1765                         "and a divisor of %u",
1766                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1767                         FM10K_RX_FREE_THRESH_MIN(q),
1768                         FM10K_RX_FREE_THRESH_DIV(q));
1769                 return -EINVAL;
1770         }
1771
1772         q->alloc_thresh = rx_free_thresh;
1773         q->drop_en = conf->rx_drop_en;
1774         q->rx_deferred_start = conf->rx_deferred_start;
1775
1776         return 0;
1777 }
1778
1779 /*
1780  * Hardware requires specific alignment for Rx packet buffers. At
1781  * least one of the following two conditions must be satisfied.
1782  *  1. Address is 512B aligned
1783  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1784  *
1785  * As such, the driver may need to adjust the DMA address within the
1786  * buffer by up to 512B.
1787  *
1788  * return 1 if the element size is valid, otherwise return 0.
1789  */
1790 static int
1791 mempool_element_size_valid(struct rte_mempool *mp)
1792 {
1793         uint32_t min_size;
1794
1795         /* elt_size includes mbuf header and headroom */
1796         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1797                         RTE_PKTMBUF_HEADROOM;
1798
1799         /* account for up to 512B of alignment */
1800         min_size -= FM10K_RX_DATABUF_ALIGN;
1801
1802         /* sanity check for overflow */
1803         if (min_size > mp->elt_size)
1804                 return 0;
1805
1806         /* size is valid */
1807         return 1;
1808 }
1809
1810 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1811 {
1812         RTE_SET_USED(dev);
1813
1814         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1815 }
1816
1817 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1818 {
1819         RTE_SET_USED(dev);
1820
1821         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1822                            DEV_RX_OFFLOAD_VLAN_FILTER |
1823                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1824                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1825                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1826                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1827                            DEV_RX_OFFLOAD_CRC_STRIP   |
1828                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1829 }
1830
1831 static int
1832 fm10k_check_rx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
1833 {
1834         uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
1835         uint64_t queue_supported = fm10k_get_rx_queue_offloads_capa(dev);
1836         uint64_t port_supported = fm10k_get_rx_port_offloads_capa(dev);
1837
1838         if ((requested & (queue_supported | port_supported)) != requested)
1839                 return 0;
1840
1841         if ((port_offloads ^ requested) & port_supported)
1842                 return 0;
1843
1844         return 1;
1845 }
1846
1847 static int
1848 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1849         uint16_t nb_desc, unsigned int socket_id,
1850         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1851 {
1852         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1853         struct fm10k_dev_info *dev_info =
1854                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1855         struct fm10k_rx_queue *q;
1856         const struct rte_memzone *mz;
1857
1858         PMD_INIT_FUNC_TRACE();
1859
1860         if (!fm10k_check_rx_queue_offloads(dev, conf->offloads)) {
1861                 PMD_INIT_LOG(ERR, "%p: Rx queue offloads 0x%" PRIx64
1862                         " don't match port offloads 0x%" PRIx64
1863                         " or supported port offloads 0x%" PRIx64
1864                         " or supported queue offloads 0x%" PRIx64,
1865                         (void *)dev, conf->offloads,
1866                         dev->data->dev_conf.rxmode.offloads,
1867                         fm10k_get_rx_port_offloads_capa(dev),
1868                         fm10k_get_rx_queue_offloads_capa(dev));
1869                 return -ENOTSUP;
1870         }
1871
1872         /* make sure the mempool element size can account for alignment. */
1873         if (!mempool_element_size_valid(mp)) {
1874                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1875                 return -EINVAL;
1876         }
1877
1878         /* make sure a valid number of descriptors have been requested */
1879         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1880                                 FM10K_MULT_RX_DESC, nb_desc)) {
1881                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1882                         "less than or equal to %"PRIu32", "
1883                         "greater than or equal to %u, "
1884                         "and a multiple of %u",
1885                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1886                         FM10K_MULT_RX_DESC);
1887                 return -EINVAL;
1888         }
1889
1890         /*
1891          * if this queue existed already, free the associated memory. The
1892          * queue cannot be reused in case we need to allocate memory on
1893          * different socket than was previously used.
1894          */
1895         if (dev->data->rx_queues[queue_id] != NULL) {
1896                 rx_queue_free(dev->data->rx_queues[queue_id]);
1897                 dev->data->rx_queues[queue_id] = NULL;
1898         }
1899
1900         /* allocate memory for the queue structure */
1901         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1902                                 socket_id);
1903         if (q == NULL) {
1904                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1905                 return -ENOMEM;
1906         }
1907
1908         /* setup queue */
1909         q->mp = mp;
1910         q->nb_desc = nb_desc;
1911         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1912         q->port_id = dev->data->port_id;
1913         q->queue_id = queue_id;
1914         q->tail_ptr = (volatile uint32_t *)
1915                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1916         q->offloads = conf->offloads;
1917         if (handle_rxconf(q, conf))
1918                 return -EINVAL;
1919
1920         /* allocate memory for the software ring */
1921         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1922                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1923                         RTE_CACHE_LINE_SIZE, socket_id);
1924         if (q->sw_ring == NULL) {
1925                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1926                 rte_free(q);
1927                 return -ENOMEM;
1928         }
1929
1930         /*
1931          * allocate memory for the hardware descriptor ring. A memzone large
1932          * enough to hold the maximum ring size is requested to allow for
1933          * resizing in later calls to the queue setup function.
1934          */
1935         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1936                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1937                                       socket_id);
1938         if (mz == NULL) {
1939                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1940                 rte_free(q->sw_ring);
1941                 rte_free(q);
1942                 return -ENOMEM;
1943         }
1944         q->hw_ring = mz->addr;
1945         q->hw_ring_phys_addr = mz->iova;
1946
1947         /* Check if number of descs satisfied Vector requirement */
1948         if (!rte_is_power_of_2(nb_desc)) {
1949                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1950                                     "preconditions - canceling the feature for "
1951                                     "the whole port[%d]",
1952                              q->queue_id, q->port_id);
1953                 dev_info->rx_vec_allowed = false;
1954         } else
1955                 fm10k_rxq_vec_setup(q);
1956
1957         dev->data->rx_queues[queue_id] = q;
1958         return 0;
1959 }
1960
1961 static void
1962 fm10k_rx_queue_release(void *queue)
1963 {
1964         PMD_INIT_FUNC_TRACE();
1965
1966         rx_queue_free(queue);
1967 }
1968
1969 static inline int
1970 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1971 {
1972         uint16_t tx_free_thresh;
1973         uint16_t tx_rs_thresh;
1974
1975         /* constraint MACROs require that tx_free_thresh is configured
1976          * before tx_rs_thresh */
1977         if (conf->tx_free_thresh == 0)
1978                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1979         else
1980                 tx_free_thresh = conf->tx_free_thresh;
1981
1982         /* make sure the requested threshold satisfies the constraints */
1983         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1984                         FM10K_TX_FREE_THRESH_MAX(q),
1985                         FM10K_TX_FREE_THRESH_DIV(q),
1986                         tx_free_thresh)) {
1987                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1988                         "less than or equal to %u, "
1989                         "greater than or equal to %u, "
1990                         "and a divisor of %u",
1991                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1992                         FM10K_TX_FREE_THRESH_MIN(q),
1993                         FM10K_TX_FREE_THRESH_DIV(q));
1994                 return -EINVAL;
1995         }
1996
1997         q->free_thresh = tx_free_thresh;
1998
1999         if (conf->tx_rs_thresh == 0)
2000                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
2001         else
2002                 tx_rs_thresh = conf->tx_rs_thresh;
2003
2004         q->tx_deferred_start = conf->tx_deferred_start;
2005
2006         /* make sure the requested threshold satisfies the constraints */
2007         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
2008                         FM10K_TX_RS_THRESH_MAX(q),
2009                         FM10K_TX_RS_THRESH_DIV(q),
2010                         tx_rs_thresh)) {
2011                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
2012                         "less than or equal to %u, "
2013                         "greater than or equal to %u, "
2014                         "and a divisor of %u",
2015                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
2016                         FM10K_TX_RS_THRESH_MIN(q),
2017                         FM10K_TX_RS_THRESH_DIV(q));
2018                 return -EINVAL;
2019         }
2020
2021         q->rs_thresh = tx_rs_thresh;
2022
2023         return 0;
2024 }
2025
2026 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
2027 {
2028         RTE_SET_USED(dev);
2029
2030         return 0;
2031 }
2032
2033 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
2034 {
2035         RTE_SET_USED(dev);
2036
2037         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
2038                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
2039                           DEV_TX_OFFLOAD_UDP_CKSUM   |
2040                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2041                           DEV_TX_OFFLOAD_TCP_TSO);
2042 }
2043
2044 static int
2045 fm10k_check_tx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
2046 {
2047         uint64_t port_offloads = dev->data->dev_conf.txmode.offloads;
2048         uint64_t queue_supported = fm10k_get_tx_queue_offloads_capa(dev);
2049         uint64_t port_supported = fm10k_get_tx_port_offloads_capa(dev);
2050
2051         if ((requested & (queue_supported | port_supported)) != requested)
2052                 return 0;
2053
2054         if ((port_offloads ^ requested) & port_supported)
2055                 return 0;
2056
2057         return 1;
2058 }
2059
2060 static int
2061 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2062         uint16_t nb_desc, unsigned int socket_id,
2063         const struct rte_eth_txconf *conf)
2064 {
2065         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2066         struct fm10k_tx_queue *q;
2067         const struct rte_memzone *mz;
2068
2069         PMD_INIT_FUNC_TRACE();
2070
2071         if (!fm10k_check_tx_queue_offloads(dev, conf->offloads)) {
2072                 PMD_INIT_LOG(ERR, "%p: Tx queue offloads 0x%" PRIx64
2073                         " don't match port offloads 0x%" PRIx64
2074                         " or supported port offloads 0x%" PRIx64
2075                         " or supported queue offloads 0x%" PRIx64,
2076                         (void *)dev, conf->offloads,
2077                         dev->data->dev_conf.txmode.offloads,
2078                         fm10k_get_tx_port_offloads_capa(dev),
2079                         fm10k_get_tx_queue_offloads_capa(dev));
2080                 return -ENOTSUP;
2081         }
2082
2083         /* make sure a valid number of descriptors have been requested */
2084         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2085                                 FM10K_MULT_TX_DESC, nb_desc)) {
2086                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2087                         "less than or equal to %"PRIu32", "
2088                         "greater than or equal to %u, "
2089                         "and a multiple of %u",
2090                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2091                         FM10K_MULT_TX_DESC);
2092                 return -EINVAL;
2093         }
2094
2095         /*
2096          * if this queue existed already, free the associated memory. The
2097          * queue cannot be reused in case we need to allocate memory on
2098          * different socket than was previously used.
2099          */
2100         if (dev->data->tx_queues[queue_id] != NULL) {
2101                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2102
2103                 tx_queue_free(txq);
2104                 dev->data->tx_queues[queue_id] = NULL;
2105         }
2106
2107         /* allocate memory for the queue structure */
2108         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2109                                 socket_id);
2110         if (q == NULL) {
2111                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2112                 return -ENOMEM;
2113         }
2114
2115         /* setup queue */
2116         q->nb_desc = nb_desc;
2117         q->port_id = dev->data->port_id;
2118         q->queue_id = queue_id;
2119         q->txq_flags = conf->txq_flags;
2120         q->offloads = conf->offloads;
2121         q->ops = &def_txq_ops;
2122         q->tail_ptr = (volatile uint32_t *)
2123                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2124         if (handle_txconf(q, conf))
2125                 return -EINVAL;
2126
2127         /* allocate memory for the software ring */
2128         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2129                                         nb_desc * sizeof(struct rte_mbuf *),
2130                                         RTE_CACHE_LINE_SIZE, socket_id);
2131         if (q->sw_ring == NULL) {
2132                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2133                 rte_free(q);
2134                 return -ENOMEM;
2135         }
2136
2137         /*
2138          * allocate memory for the hardware descriptor ring. A memzone large
2139          * enough to hold the maximum ring size is requested to allow for
2140          * resizing in later calls to the queue setup function.
2141          */
2142         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2143                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2144                                       socket_id);
2145         if (mz == NULL) {
2146                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2147                 rte_free(q->sw_ring);
2148                 rte_free(q);
2149                 return -ENOMEM;
2150         }
2151         q->hw_ring = mz->addr;
2152         q->hw_ring_phys_addr = mz->iova;
2153
2154         /*
2155          * allocate memory for the RS bit tracker. Enough slots to hold the
2156          * descriptor index for each RS bit needing to be set are required.
2157          */
2158         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2159                                 ((nb_desc + 1) / q->rs_thresh) *
2160                                 sizeof(uint16_t),
2161                                 RTE_CACHE_LINE_SIZE, socket_id);
2162         if (q->rs_tracker.list == NULL) {
2163                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2164                 rte_free(q->sw_ring);
2165                 rte_free(q);
2166                 return -ENOMEM;
2167         }
2168
2169         dev->data->tx_queues[queue_id] = q;
2170         return 0;
2171 }
2172
2173 static void
2174 fm10k_tx_queue_release(void *queue)
2175 {
2176         struct fm10k_tx_queue *q = queue;
2177         PMD_INIT_FUNC_TRACE();
2178
2179         tx_queue_free(q);
2180 }
2181
2182 static int
2183 fm10k_reta_update(struct rte_eth_dev *dev,
2184                         struct rte_eth_rss_reta_entry64 *reta_conf,
2185                         uint16_t reta_size)
2186 {
2187         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2188         uint16_t i, j, idx, shift;
2189         uint8_t mask;
2190         uint32_t reta;
2191
2192         PMD_INIT_FUNC_TRACE();
2193
2194         if (reta_size > FM10K_MAX_RSS_INDICES) {
2195                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2196                         "(%d) doesn't match the number hardware can supported "
2197                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2198                 return -EINVAL;
2199         }
2200
2201         /*
2202          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2203          * 128-entries in 32 registers
2204          */
2205         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2206                 idx = i / RTE_RETA_GROUP_SIZE;
2207                 shift = i % RTE_RETA_GROUP_SIZE;
2208                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2209                                 BIT_MASK_PER_UINT32);
2210                 if (mask == 0)
2211                         continue;
2212
2213                 reta = 0;
2214                 if (mask != BIT_MASK_PER_UINT32)
2215                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2216
2217                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2218                         if (mask & (0x1 << j)) {
2219                                 if (mask != 0xF)
2220                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2221                                 reta |= reta_conf[idx].reta[shift + j] <<
2222                                                 (CHAR_BIT * j);
2223                         }
2224                 }
2225                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2226         }
2227
2228         return 0;
2229 }
2230
2231 static int
2232 fm10k_reta_query(struct rte_eth_dev *dev,
2233                         struct rte_eth_rss_reta_entry64 *reta_conf,
2234                         uint16_t reta_size)
2235 {
2236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2237         uint16_t i, j, idx, shift;
2238         uint8_t mask;
2239         uint32_t reta;
2240
2241         PMD_INIT_FUNC_TRACE();
2242
2243         if (reta_size < FM10K_MAX_RSS_INDICES) {
2244                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2245                         "(%d) doesn't match the number hardware can supported "
2246                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2247                 return -EINVAL;
2248         }
2249
2250         /*
2251          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2252          * 128-entries in 32 registers
2253          */
2254         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2255                 idx = i / RTE_RETA_GROUP_SIZE;
2256                 shift = i % RTE_RETA_GROUP_SIZE;
2257                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2258                                 BIT_MASK_PER_UINT32);
2259                 if (mask == 0)
2260                         continue;
2261
2262                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2263                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2264                         if (mask & (0x1 << j))
2265                                 reta_conf[idx].reta[shift + j] = ((reta >>
2266                                         CHAR_BIT * j) & UINT8_MAX);
2267                 }
2268         }
2269
2270         return 0;
2271 }
2272
2273 static int
2274 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2275         struct rte_eth_rss_conf *rss_conf)
2276 {
2277         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2278         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2279         uint32_t mrqc;
2280         uint64_t hf = rss_conf->rss_hf;
2281         int i;
2282
2283         PMD_INIT_FUNC_TRACE();
2284
2285         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2286                                 FM10K_RSSRK_ENTRIES_PER_REG))
2287                 return -EINVAL;
2288
2289         if (hf == 0)
2290                 return -EINVAL;
2291
2292         mrqc = 0;
2293         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2294         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2295         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2296         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2297         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2298         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2299         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2300         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2301         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2302
2303         /* If the mapping doesn't fit any supported, return */
2304         if (mrqc == 0)
2305                 return -EINVAL;
2306
2307         if (key != NULL)
2308                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2309                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2310
2311         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2312
2313         return 0;
2314 }
2315
2316 static int
2317 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2318         struct rte_eth_rss_conf *rss_conf)
2319 {
2320         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2321         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2322         uint32_t mrqc;
2323         uint64_t hf;
2324         int i;
2325
2326         PMD_INIT_FUNC_TRACE();
2327
2328         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2329                                 FM10K_RSSRK_ENTRIES_PER_REG))
2330                 return -EINVAL;
2331
2332         if (key != NULL)
2333                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2334                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2335
2336         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2337         hf = 0;
2338         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2339         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2340         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2341         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2342         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2343         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2344         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2345         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2346         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2347
2348         rss_conf->rss_hf = hf;
2349
2350         return 0;
2351 }
2352
2353 static void
2354 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2355 {
2356         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2357         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2358
2359         /* Bind all local non-queue interrupt to vector 0 */
2360         int_map |= FM10K_MISC_VEC_ID;
2361
2362         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2363         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2364         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2365         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2366         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2367         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2368
2369         /* Enable misc causes */
2370         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2371                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2372                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2373                                 FM10K_EIMR_ENABLE(MAILBOX) |
2374                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2375                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2376                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2377                                 FM10K_EIMR_ENABLE(VFLR));
2378
2379         /* Enable ITR 0 */
2380         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2381                                         FM10K_ITR_MASK_CLEAR);
2382         FM10K_WRITE_FLUSH(hw);
2383 }
2384
2385 static void
2386 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2387 {
2388         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2389         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2390
2391         int_map |= FM10K_MISC_VEC_ID;
2392
2393         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2394         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2395         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2396         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2397         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2398         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2399
2400         /* Disable misc causes */
2401         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2402                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2403                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2404                                 FM10K_EIMR_DISABLE(MAILBOX) |
2405                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2406                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2407                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2408                                 FM10K_EIMR_DISABLE(VFLR));
2409
2410         /* Disable ITR 0 */
2411         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2412         FM10K_WRITE_FLUSH(hw);
2413 }
2414
2415 static void
2416 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2417 {
2418         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2419         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2420
2421         /* Bind all local non-queue interrupt to vector 0 */
2422         int_map |= FM10K_MISC_VEC_ID;
2423
2424         /* Only INT 0 available, other 15 are reserved. */
2425         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2426
2427         /* Enable ITR 0 */
2428         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2429                                         FM10K_ITR_MASK_CLEAR);
2430         FM10K_WRITE_FLUSH(hw);
2431 }
2432
2433 static void
2434 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2435 {
2436         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2437         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2438
2439         int_map |= FM10K_MISC_VEC_ID;
2440
2441         /* Only INT 0 available, other 15 are reserved. */
2442         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2443
2444         /* Disable ITR 0 */
2445         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2446         FM10K_WRITE_FLUSH(hw);
2447 }
2448
2449 static int
2450 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2451 {
2452         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2453         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2454
2455         /* Enable ITR */
2456         if (hw->mac.type == fm10k_mac_pf)
2457                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2458                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2459         else
2460                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2461                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2462         rte_intr_enable(&pdev->intr_handle);
2463         return 0;
2464 }
2465
2466 static int
2467 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2468 {
2469         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2470         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2471
2472         /* Disable ITR */
2473         if (hw->mac.type == fm10k_mac_pf)
2474                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2475                         FM10K_ITR_MASK_SET);
2476         else
2477                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2478                         FM10K_ITR_MASK_SET);
2479         return 0;
2480 }
2481
2482 static int
2483 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2484 {
2485         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2486         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2487         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2488         uint32_t intr_vector, vec;
2489         uint16_t queue_id;
2490         int result = 0;
2491
2492         /* fm10k needs one separate interrupt for mailbox,
2493          * so only drivers which support multiple interrupt vectors
2494          * e.g. vfio-pci can work for fm10k interrupt mode
2495          */
2496         if (!rte_intr_cap_multiple(intr_handle) ||
2497                         dev->data->dev_conf.intr_conf.rxq == 0)
2498                 return result;
2499
2500         intr_vector = dev->data->nb_rx_queues;
2501
2502         /* disable interrupt first */
2503         rte_intr_disable(intr_handle);
2504         if (hw->mac.type == fm10k_mac_pf)
2505                 fm10k_dev_disable_intr_pf(dev);
2506         else
2507                 fm10k_dev_disable_intr_vf(dev);
2508
2509         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2510                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2511                 result = -EIO;
2512         }
2513
2514         if (rte_intr_dp_is_en(intr_handle) && !result) {
2515                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2516                         dev->data->nb_rx_queues * sizeof(int), 0);
2517                 if (intr_handle->intr_vec) {
2518                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2519                                         queue_id < dev->data->nb_rx_queues;
2520                                         queue_id++) {
2521                                 intr_handle->intr_vec[queue_id] = vec;
2522                                 if (vec < intr_handle->nb_efd - 1
2523                                                 + FM10K_RX_VEC_START)
2524                                         vec++;
2525                         }
2526                 } else {
2527                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2528                                 " intr_vec", dev->data->nb_rx_queues);
2529                         rte_intr_efd_disable(intr_handle);
2530                         result = -ENOMEM;
2531                 }
2532         }
2533
2534         if (hw->mac.type == fm10k_mac_pf)
2535                 fm10k_dev_enable_intr_pf(dev);
2536         else
2537                 fm10k_dev_enable_intr_vf(dev);
2538         rte_intr_enable(intr_handle);
2539         hw->mac.ops.update_int_moderator(hw);
2540         return result;
2541 }
2542
2543 static int
2544 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2545 {
2546         struct fm10k_fault fault;
2547         int err;
2548         const char *estr = "Unknown error";
2549
2550         /* Process PCA fault */
2551         if (eicr & FM10K_EICR_PCA_FAULT) {
2552                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2553                 if (err)
2554                         goto error;
2555                 switch (fault.type) {
2556                 case PCA_NO_FAULT:
2557                         estr = "PCA_NO_FAULT"; break;
2558                 case PCA_UNMAPPED_ADDR:
2559                         estr = "PCA_UNMAPPED_ADDR"; break;
2560                 case PCA_BAD_QACCESS_PF:
2561                         estr = "PCA_BAD_QACCESS_PF"; break;
2562                 case PCA_BAD_QACCESS_VF:
2563                         estr = "PCA_BAD_QACCESS_VF"; break;
2564                 case PCA_MALICIOUS_REQ:
2565                         estr = "PCA_MALICIOUS_REQ"; break;
2566                 case PCA_POISONED_TLP:
2567                         estr = "PCA_POISONED_TLP"; break;
2568                 case PCA_TLP_ABORT:
2569                         estr = "PCA_TLP_ABORT"; break;
2570                 default:
2571                         goto error;
2572                 }
2573                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2574                         estr, fault.func ? "VF" : "PF", fault.func,
2575                         fault.address, fault.specinfo);
2576         }
2577
2578         /* Process THI fault */
2579         if (eicr & FM10K_EICR_THI_FAULT) {
2580                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2581                 if (err)
2582                         goto error;
2583                 switch (fault.type) {
2584                 case THI_NO_FAULT:
2585                         estr = "THI_NO_FAULT"; break;
2586                 case THI_MAL_DIS_Q_FAULT:
2587                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2588                 default:
2589                         goto error;
2590                 }
2591                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2592                         estr, fault.func ? "VF" : "PF", fault.func,
2593                         fault.address, fault.specinfo);
2594         }
2595
2596         /* Process FUM fault */
2597         if (eicr & FM10K_EICR_FUM_FAULT) {
2598                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2599                 if (err)
2600                         goto error;
2601                 switch (fault.type) {
2602                 case FUM_NO_FAULT:
2603                         estr = "FUM_NO_FAULT"; break;
2604                 case FUM_UNMAPPED_ADDR:
2605                         estr = "FUM_UNMAPPED_ADDR"; break;
2606                 case FUM_POISONED_TLP:
2607                         estr = "FUM_POISONED_TLP"; break;
2608                 case FUM_BAD_VF_QACCESS:
2609                         estr = "FUM_BAD_VF_QACCESS"; break;
2610                 case FUM_ADD_DECODE_ERR:
2611                         estr = "FUM_ADD_DECODE_ERR"; break;
2612                 case FUM_RO_ERROR:
2613                         estr = "FUM_RO_ERROR"; break;
2614                 case FUM_QPRC_CRC_ERROR:
2615                         estr = "FUM_QPRC_CRC_ERROR"; break;
2616                 case FUM_CSR_TIMEOUT:
2617                         estr = "FUM_CSR_TIMEOUT"; break;
2618                 case FUM_INVALID_TYPE:
2619                         estr = "FUM_INVALID_TYPE"; break;
2620                 case FUM_INVALID_LENGTH:
2621                         estr = "FUM_INVALID_LENGTH"; break;
2622                 case FUM_INVALID_BE:
2623                         estr = "FUM_INVALID_BE"; break;
2624                 case FUM_INVALID_ALIGN:
2625                         estr = "FUM_INVALID_ALIGN"; break;
2626                 default:
2627                         goto error;
2628                 }
2629                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2630                         estr, fault.func ? "VF" : "PF", fault.func,
2631                         fault.address, fault.specinfo);
2632         }
2633
2634         return 0;
2635 error:
2636         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2637         return err;
2638 }
2639
2640 /**
2641  * PF interrupt handler triggered by NIC for handling specific interrupt.
2642  *
2643  * @param handle
2644  *  Pointer to interrupt handle.
2645  * @param param
2646  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2647  *
2648  * @return
2649  *  void
2650  */
2651 static void
2652 fm10k_dev_interrupt_handler_pf(void *param)
2653 {
2654         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2655         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2656         uint32_t cause, status;
2657         struct fm10k_dev_info *dev_info =
2658                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2659         int status_mbx;
2660         s32 err;
2661
2662         if (hw->mac.type != fm10k_mac_pf)
2663                 return;
2664
2665         cause = FM10K_READ_REG(hw, FM10K_EICR);
2666
2667         /* Handle PCI fault cases */
2668         if (cause & FM10K_EICR_FAULT_MASK) {
2669                 PMD_INIT_LOG(ERR, "INT: find fault!");
2670                 fm10k_dev_handle_fault(hw, cause);
2671         }
2672
2673         /* Handle switch up/down */
2674         if (cause & FM10K_EICR_SWITCHNOTREADY)
2675                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2676
2677         if (cause & FM10K_EICR_SWITCHREADY) {
2678                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2679                 if (dev_info->sm_down == 1) {
2680                         fm10k_mbx_lock(hw);
2681
2682                         /* For recreating logical ports */
2683                         status_mbx = hw->mac.ops.update_lport_state(hw,
2684                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2685                         if (status_mbx == FM10K_SUCCESS)
2686                                 PMD_INIT_LOG(INFO,
2687                                         "INT: Recreated Logical port");
2688                         else
2689                                 PMD_INIT_LOG(INFO,
2690                                         "INT: Logical ports weren't recreated");
2691
2692                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2693                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2694                         if (status_mbx != FM10K_SUCCESS)
2695                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2696
2697                         fm10k_mbx_unlock(hw);
2698
2699                         /* first clear the internal SW recording structure */
2700                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2701                                                 ETH_MQ_RX_VMDQ_FLAG))
2702                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2703                                         false);
2704
2705                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2706                                         MAIN_VSI_POOL_NUMBER);
2707
2708                         /*
2709                          * Add default mac address and vlan for the logical
2710                          * ports that have been created, leave to the
2711                          * application to fully recover Rx filtering.
2712                          */
2713                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2714                                         MAIN_VSI_POOL_NUMBER);
2715
2716                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2717                                                 ETH_MQ_RX_VMDQ_FLAG))
2718                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2719                                         true);
2720
2721                         dev_info->sm_down = 0;
2722                         _rte_eth_dev_callback_process(dev,
2723                                         RTE_ETH_EVENT_INTR_LSC,
2724                                         NULL);
2725                 }
2726         }
2727
2728         /* Handle mailbox message */
2729         fm10k_mbx_lock(hw);
2730         err = hw->mbx.ops.process(hw, &hw->mbx);
2731         fm10k_mbx_unlock(hw);
2732
2733         if (err == FM10K_ERR_RESET_REQUESTED) {
2734                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2735                 dev_info->sm_down = 1;
2736                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2737                                 NULL);
2738         }
2739
2740         /* Handle SRAM error */
2741         if (cause & FM10K_EICR_SRAMERROR) {
2742                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2743
2744                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2745                 /* Write to clear pending bits */
2746                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2747
2748                 /* Todo: print out error message after shared code  updates */
2749         }
2750
2751         /* Clear these 3 events if having any */
2752         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2753                  FM10K_EICR_SWITCHREADY;
2754         if (cause)
2755                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2756
2757         /* Re-enable interrupt from device side */
2758         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2759                                         FM10K_ITR_MASK_CLEAR);
2760         /* Re-enable interrupt from host side */
2761         rte_intr_enable(dev->intr_handle);
2762 }
2763
2764 /**
2765  * VF interrupt handler triggered by NIC for handling specific interrupt.
2766  *
2767  * @param handle
2768  *  Pointer to interrupt handle.
2769  * @param param
2770  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2771  *
2772  * @return
2773  *  void
2774  */
2775 static void
2776 fm10k_dev_interrupt_handler_vf(void *param)
2777 {
2778         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2779         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2780         struct fm10k_mbx_info *mbx = &hw->mbx;
2781         struct fm10k_dev_info *dev_info =
2782                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2783         const enum fm10k_mbx_state state = mbx->state;
2784         int status_mbx;
2785
2786         if (hw->mac.type != fm10k_mac_vf)
2787                 return;
2788
2789         /* Handle mailbox message if lock is acquired */
2790         fm10k_mbx_lock(hw);
2791         hw->mbx.ops.process(hw, &hw->mbx);
2792         fm10k_mbx_unlock(hw);
2793
2794         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2795                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2796
2797                 fm10k_mbx_lock(hw);
2798                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2799                                 MAX_LPORT_NUM, 1);
2800                 fm10k_mbx_unlock(hw);
2801
2802                 /* Setting reset flag */
2803                 dev_info->sm_down = 1;
2804                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2805                                 NULL);
2806         }
2807
2808         if (dev_info->sm_down == 1 &&
2809                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2810                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2811                 fm10k_mbx_lock(hw);
2812                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2813                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2814                 if (status_mbx != FM10K_SUCCESS)
2815                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2816                 fm10k_mbx_unlock(hw);
2817
2818                 /* first clear the internal SW recording structure */
2819                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2820                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2821                                 MAIN_VSI_POOL_NUMBER);
2822
2823                 /*
2824                  * Add default mac address and vlan for the logical ports that
2825                  * have been created, leave to the application to fully recover
2826                  * Rx filtering.
2827                  */
2828                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2829                                 MAIN_VSI_POOL_NUMBER);
2830                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2831
2832                 dev_info->sm_down = 0;
2833                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2834                                 NULL);
2835         }
2836
2837         /* Re-enable interrupt from device side */
2838         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2839                                         FM10K_ITR_MASK_CLEAR);
2840         /* Re-enable interrupt from host side */
2841         rte_intr_enable(dev->intr_handle);
2842 }
2843
2844 /* Mailbox message handler in VF */
2845 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2846         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2847         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2848         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2849         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2850 };
2851
2852 static int
2853 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2854 {
2855         int err = 0;
2856
2857         /* Initialize mailbox lock */
2858         fm10k_mbx_initlock(hw);
2859
2860         /* Replace default message handler with new ones */
2861         if (hw->mac.type == fm10k_mac_vf)
2862                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2863
2864         if (err) {
2865                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2866                                 err);
2867                 return err;
2868         }
2869         /* Connect to SM for PF device or PF for VF device */
2870         return hw->mbx.ops.connect(hw, &hw->mbx);
2871 }
2872
2873 static void
2874 fm10k_close_mbx_service(struct fm10k_hw *hw)
2875 {
2876         /* Disconnect from SM for PF device or PF for VF device */
2877         hw->mbx.ops.disconnect(hw, &hw->mbx);
2878 }
2879
2880 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2881         .dev_configure          = fm10k_dev_configure,
2882         .dev_start              = fm10k_dev_start,
2883         .dev_stop               = fm10k_dev_stop,
2884         .dev_close              = fm10k_dev_close,
2885         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2886         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2887         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2888         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2889         .stats_get              = fm10k_stats_get,
2890         .xstats_get             = fm10k_xstats_get,
2891         .xstats_get_names       = fm10k_xstats_get_names,
2892         .stats_reset            = fm10k_stats_reset,
2893         .xstats_reset           = fm10k_stats_reset,
2894         .link_update            = fm10k_link_update,
2895         .dev_infos_get          = fm10k_dev_infos_get,
2896         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2897         .vlan_filter_set        = fm10k_vlan_filter_set,
2898         .vlan_offload_set       = fm10k_vlan_offload_set,
2899         .mac_addr_add           = fm10k_macaddr_add,
2900         .mac_addr_remove        = fm10k_macaddr_remove,
2901         .rx_queue_start         = fm10k_dev_rx_queue_start,
2902         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2903         .tx_queue_start         = fm10k_dev_tx_queue_start,
2904         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2905         .rx_queue_setup         = fm10k_rx_queue_setup,
2906         .rx_queue_release       = fm10k_rx_queue_release,
2907         .tx_queue_setup         = fm10k_tx_queue_setup,
2908         .tx_queue_release       = fm10k_tx_queue_release,
2909         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2910         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2911         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2912         .reta_update            = fm10k_reta_update,
2913         .reta_query             = fm10k_reta_query,
2914         .rss_hash_update        = fm10k_rss_hash_update,
2915         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2916 };
2917
2918 static int ftag_check_handler(__rte_unused const char *key,
2919                 const char *value, __rte_unused void *opaque)
2920 {
2921         if (strcmp(value, "1"))
2922                 return -1;
2923
2924         return 0;
2925 }
2926
2927 static int
2928 fm10k_check_ftag(struct rte_devargs *devargs)
2929 {
2930         struct rte_kvargs *kvlist;
2931         const char *ftag_key = "enable_ftag";
2932
2933         if (devargs == NULL)
2934                 return 0;
2935
2936         kvlist = rte_kvargs_parse(devargs->args, NULL);
2937         if (kvlist == NULL)
2938                 return 0;
2939
2940         if (!rte_kvargs_count(kvlist, ftag_key)) {
2941                 rte_kvargs_free(kvlist);
2942                 return 0;
2943         }
2944         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2945         if (rte_kvargs_process(kvlist, ftag_key,
2946                                 ftag_check_handler, NULL) < 0) {
2947                 rte_kvargs_free(kvlist);
2948                 return 0;
2949         }
2950         rte_kvargs_free(kvlist);
2951
2952         return 1;
2953 }
2954
2955 static uint16_t
2956 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2957                     uint16_t nb_pkts)
2958 {
2959         uint16_t nb_tx = 0;
2960         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2961
2962         while (nb_pkts) {
2963                 uint16_t ret, num;
2964
2965                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2966                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2967                                                  num);
2968                 nb_tx += ret;
2969                 nb_pkts -= ret;
2970                 if (ret < num)
2971                         break;
2972         }
2973
2974         return nb_tx;
2975 }
2976
2977 static void __attribute__((cold))
2978 fm10k_set_tx_function(struct rte_eth_dev *dev)
2979 {
2980         struct fm10k_tx_queue *txq;
2981         int i;
2982         int use_sse = 1;
2983         uint16_t tx_ftag_en = 0;
2984
2985         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2986                 /* primary process has set the ftag flag and txq_flags */
2987                 txq = dev->data->tx_queues[0];
2988                 if (fm10k_tx_vec_condition_check(txq)) {
2989                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2990                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2991                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2992                 } else {
2993                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2994                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2995                         dev->tx_pkt_prepare = NULL;
2996                 }
2997                 return;
2998         }
2999
3000         if (fm10k_check_ftag(dev->device->devargs))
3001                 tx_ftag_en = 1;
3002
3003         for (i = 0; i < dev->data->nb_tx_queues; i++) {
3004                 txq = dev->data->tx_queues[i];
3005                 txq->tx_ftag_en = tx_ftag_en;
3006                 /* Check if Vector Tx is satisfied */
3007                 if (fm10k_tx_vec_condition_check(txq))
3008                         use_sse = 0;
3009         }
3010
3011         if (use_sse) {
3012                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
3013                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3014                         txq = dev->data->tx_queues[i];
3015                         fm10k_txq_vec_setup(txq);
3016                 }
3017                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
3018                 dev->tx_pkt_prepare = NULL;
3019         } else {
3020                 dev->tx_pkt_burst = fm10k_xmit_pkts;
3021                 dev->tx_pkt_prepare = fm10k_prep_pkts;
3022                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
3023         }
3024 }
3025
3026 static void __attribute__((cold))
3027 fm10k_set_rx_function(struct rte_eth_dev *dev)
3028 {
3029         struct fm10k_dev_info *dev_info =
3030                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3031         uint16_t i, rx_using_sse;
3032         uint16_t rx_ftag_en = 0;
3033
3034         if (fm10k_check_ftag(dev->device->devargs))
3035                 rx_ftag_en = 1;
3036
3037         /* In order to allow Vector Rx there are a few configuration
3038          * conditions to be met.
3039          */
3040         if (!fm10k_rx_vec_condition_check(dev) &&
3041                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3042                 if (dev->data->scattered_rx)
3043                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3044                 else
3045                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3046         } else if (dev->data->scattered_rx)
3047                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3048         else
3049                 dev->rx_pkt_burst = fm10k_recv_pkts;
3050
3051         rx_using_sse =
3052                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3053                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3054
3055         if (rx_using_sse)
3056                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3057         else
3058                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3059
3060         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3061                 return;
3062
3063         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3064                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3065
3066                 rxq->rx_using_sse = rx_using_sse;
3067                 rxq->rx_ftag_en = rx_ftag_en;
3068         }
3069 }
3070
3071 static void
3072 fm10k_params_init(struct rte_eth_dev *dev)
3073 {
3074         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3075         struct fm10k_dev_info *info =
3076                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3077
3078         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3079          * there is no way to get link status without reading BAR4.  Until this
3080          * works, assume we have maximum bandwidth.
3081          * @todo - fix bus info
3082          */
3083         hw->bus_caps.speed = fm10k_bus_speed_8000;
3084         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3085         hw->bus_caps.payload = fm10k_bus_payload_512;
3086         hw->bus.speed = fm10k_bus_speed_8000;
3087         hw->bus.width = fm10k_bus_width_pcie_x8;
3088         hw->bus.payload = fm10k_bus_payload_256;
3089
3090         info->rx_vec_allowed = true;
3091 }
3092
3093 static int
3094 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3095 {
3096         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3097         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3098         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3099         int diag, i;
3100         struct fm10k_macvlan_filter_info *macvlan;
3101
3102         PMD_INIT_FUNC_TRACE();
3103
3104         dev->dev_ops = &fm10k_eth_dev_ops;
3105         dev->rx_pkt_burst = &fm10k_recv_pkts;
3106         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3107         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3108
3109         /*
3110          * Primary process does the whole initialization, for secondary
3111          * processes, we just select the same Rx and Tx function as primary.
3112          */
3113         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3114                 fm10k_set_rx_function(dev);
3115                 fm10k_set_tx_function(dev);
3116                 return 0;
3117         }
3118
3119         rte_eth_copy_pci_info(dev, pdev);
3120
3121         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3122         memset(macvlan, 0, sizeof(*macvlan));
3123         /* Vendor and Device ID need to be set before init of shared code */
3124         memset(hw, 0, sizeof(*hw));
3125         hw->device_id = pdev->id.device_id;
3126         hw->vendor_id = pdev->id.vendor_id;
3127         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3128         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3129         hw->revision_id = 0;
3130         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3131         if (hw->hw_addr == NULL) {
3132                 PMD_INIT_LOG(ERR, "Bad mem resource."
3133                         " Try to blacklist unused devices.");
3134                 return -EIO;
3135         }
3136
3137         /* Store fm10k_adapter pointer */
3138         hw->back = dev->data->dev_private;
3139
3140         /* Initialize the shared code */
3141         diag = fm10k_init_shared_code(hw);
3142         if (diag != FM10K_SUCCESS) {
3143                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3144                 return -EIO;
3145         }
3146
3147         /* Initialize parameters */
3148         fm10k_params_init(dev);
3149
3150         /* Initialize the hw */
3151         diag = fm10k_init_hw(hw);
3152         if (diag != FM10K_SUCCESS) {
3153                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3154                 return -EIO;
3155         }
3156
3157         /* Initialize MAC address(es) */
3158         dev->data->mac_addrs = rte_zmalloc("fm10k",
3159                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3160         if (dev->data->mac_addrs == NULL) {
3161                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3162                 return -ENOMEM;
3163         }
3164
3165         diag = fm10k_read_mac_addr(hw);
3166
3167         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3168                         &dev->data->mac_addrs[0]);
3169
3170         if (diag != FM10K_SUCCESS ||
3171                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3172
3173                 /* Generate a random addr */
3174                 eth_random_addr(hw->mac.addr);
3175                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3176                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3177                 &dev->data->mac_addrs[0]);
3178         }
3179
3180         /* Reset the hw statistics */
3181         fm10k_stats_reset(dev);
3182
3183         /* Reset the hw */
3184         diag = fm10k_reset_hw(hw);
3185         if (diag != FM10K_SUCCESS) {
3186                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3187                 return -EIO;
3188         }
3189
3190         /* Setup mailbox service */
3191         diag = fm10k_setup_mbx_service(hw);
3192         if (diag != FM10K_SUCCESS) {
3193                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3194                 return -EIO;
3195         }
3196
3197         /*PF/VF has different interrupt handling mechanism */
3198         if (hw->mac.type == fm10k_mac_pf) {
3199                 /* register callback func to eal lib */
3200                 rte_intr_callback_register(intr_handle,
3201                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3202
3203                 /* enable MISC interrupt */
3204                 fm10k_dev_enable_intr_pf(dev);
3205         } else { /* VF */
3206                 rte_intr_callback_register(intr_handle,
3207                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3208
3209                 fm10k_dev_enable_intr_vf(dev);
3210         }
3211
3212         /* Enable intr after callback registered */
3213         rte_intr_enable(intr_handle);
3214
3215         hw->mac.ops.update_int_moderator(hw);
3216
3217         /* Make sure Switch Manager is ready before going forward. */
3218         if (hw->mac.type == fm10k_mac_pf) {
3219                 int switch_ready = 0;
3220
3221                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3222                         fm10k_mbx_lock(hw);
3223                         hw->mac.ops.get_host_state(hw, &switch_ready);
3224                         fm10k_mbx_unlock(hw);
3225                         if (switch_ready)
3226                                 break;
3227                         /* Delay some time to acquire async LPORT_MAP info. */
3228                         rte_delay_us(WAIT_SWITCH_MSG_US);
3229                 }
3230
3231                 if (switch_ready == 0) {
3232                         PMD_INIT_LOG(ERR, "switch is not ready");
3233                         return -1;
3234                 }
3235         }
3236
3237         /*
3238          * Below function will trigger operations on mailbox, acquire lock to
3239          * avoid race condition from interrupt handler. Operations on mailbox
3240          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3241          * will handle and generate an interrupt to our side. Then,  FIFO in
3242          * mailbox will be touched.
3243          */
3244         fm10k_mbx_lock(hw);
3245         /* Enable port first */
3246         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3247                                         MAX_LPORT_NUM, 1);
3248
3249         /* Set unicast mode by default. App can change to other mode in other
3250          * API func.
3251          */
3252         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3253                                         FM10K_XCAST_MODE_NONE);
3254
3255         fm10k_mbx_unlock(hw);
3256
3257         /* Make sure default VID is ready before going forward. */
3258         if (hw->mac.type == fm10k_mac_pf) {
3259                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3260                         if (hw->mac.default_vid)
3261                                 break;
3262                         /* Delay some time to acquire async port VLAN info. */
3263                         rte_delay_us(WAIT_SWITCH_MSG_US);
3264                 }
3265
3266                 if (!hw->mac.default_vid) {
3267                         PMD_INIT_LOG(ERR, "default VID is not ready");
3268                         return -1;
3269                 }
3270         }
3271
3272         /* Add default mac address */
3273         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3274                 MAIN_VSI_POOL_NUMBER);
3275
3276         return 0;
3277 }
3278
3279 static int
3280 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3281 {
3282         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3283         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3284         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3285         PMD_INIT_FUNC_TRACE();
3286
3287         /* only uninitialize in the primary process */
3288         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3289                 return 0;
3290
3291         /* safe to close dev here */
3292         fm10k_dev_close(dev);
3293
3294         dev->dev_ops = NULL;
3295         dev->rx_pkt_burst = NULL;
3296         dev->tx_pkt_burst = NULL;
3297
3298         /* disable uio/vfio intr */
3299         rte_intr_disable(intr_handle);
3300
3301         /*PF/VF has different interrupt handling mechanism */
3302         if (hw->mac.type == fm10k_mac_pf) {
3303                 /* disable interrupt */
3304                 fm10k_dev_disable_intr_pf(dev);
3305
3306                 /* unregister callback func to eal lib */
3307                 rte_intr_callback_unregister(intr_handle,
3308                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3309         } else {
3310                 /* disable interrupt */
3311                 fm10k_dev_disable_intr_vf(dev);
3312
3313                 rte_intr_callback_unregister(intr_handle,
3314                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3315         }
3316
3317         /* free mac memory */
3318         if (dev->data->mac_addrs) {
3319                 rte_free(dev->data->mac_addrs);
3320                 dev->data->mac_addrs = NULL;
3321         }
3322
3323         memset(hw, 0, sizeof(*hw));
3324
3325         return 0;
3326 }
3327
3328 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3329         struct rte_pci_device *pci_dev)
3330 {
3331         return rte_eth_dev_pci_generic_probe(pci_dev,
3332                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3333 }
3334
3335 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3336 {
3337         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3338 }
3339
3340 /*
3341  * The set of PCI devices this driver supports. This driver will enable both PF
3342  * and SRIOV-VF devices.
3343  */
3344 static const struct rte_pci_id pci_id_fm10k_map[] = {
3345         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3346         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3347         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3348         { .vendor_id = 0, /* sentinel */ },
3349 };
3350
3351 static struct rte_pci_driver rte_pmd_fm10k = {
3352         .id_table = pci_id_fm10k_map,
3353         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3354                      RTE_PCI_DRV_IOVA_AS_VA,
3355         .probe = eth_fm10k_pci_probe,
3356         .remove = eth_fm10k_pci_remove,
3357 };
3358
3359 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3360 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3361 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3362
3363 RTE_INIT(fm10k_init_log);
3364 static void
3365 fm10k_init_log(void)
3366 {
3367         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3368         if (fm10k_logtype_init >= 0)
3369                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3370         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3371         if (fm10k_logtype_driver >= 0)
3372                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3373 }