drivers/net/fm10k/fm10k_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2013-2016 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_string_fns.h>
  10 #include <rte_dev.h>
  11 #include <rte_spinlock.h>
  12 #include <rte_kvargs.h>
  13
  14 #include "fm10k.h"
  15 #include "base/fm10k_api.h"
  16
  17 /* Default delay to acquire mailbox lock */
  18 #define FM10K_MBXLOCK_DELAY_US 20
  19 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  20
  21 #define MAIN_VSI_POOL_NUMBER 0
  22
  23 /* Max try times to acquire switch status */
  24 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  25 /* Wait interval to get switch status */
  26 #define WAIT_SWITCH_MSG_US    100000
  27 /* A period of quiescence for switch */
  28 #define FM10K_SWITCH_QUIESCE_US 100000
  29 /* Number of chars per uint32 type */
  30 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  31 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  32
  33 /* default 1:1 map from queue ID to interrupt vector ID */
  34 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id])
  35
  36 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
  37 #define MAX_LPORT_NUM    128
  38 #define GLORT_FD_Q_BASE  0x40
  39 #define GLORT_PF_MASK    0xFFC0
  40 #define GLORT_FD_MASK    GLORT_PF_MASK
  41 #define GLORT_FD_INDEX   GLORT_FD_Q_BASE
  42
  43 int fm10k_logtype_init;
  44 int fm10k_logtype_driver;
  45
  46 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  47 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  48 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  49 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  50 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  51 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  52 static int
  53 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  54 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  55         const u8 *mac, bool add, uint32_t pool);
  56 static void fm10k_tx_queue_release(void *queue);
  57 static void fm10k_rx_queue_release(void *queue);
  58 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  59 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  60 static int fm10k_check_ftag(struct rte_devargs *devargs);
  61 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete);
  62
  63 static void fm10k_dev_infos_get(struct rte_eth_dev *dev,
  64                                 struct rte_eth_dev_info *dev_info);
  65 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev);
  66 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev);
  67 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev);
  68 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev);
  69
  70 struct fm10k_xstats_name_off {
  71         char name[RTE_ETH_XSTATS_NAME_SIZE];
  72         unsigned offset;
  73 };
  74
  75 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
  76         {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
  77         {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
  78         {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
  79         {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
  80         {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
  81         {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
  82         {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
  84                 nodesc_drop)},
  85 };
  86
  87 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
  88                 sizeof(fm10k_hw_stats_strings[0]))
  89
  90 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
  91         {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
  92         {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
  93         {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
  94 };
  95
  96 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
  97                 sizeof(fm10k_hw_stats_rx_q_strings[0]))
  98
  99 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
 100         {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
 101         {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
 102 };
 103
 104 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
 105                 sizeof(fm10k_hw_stats_tx_q_strings[0]))
 106
 107 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
 108                 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
 109 static int
 110 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111
 112 static void
 113 fm10k_mbx_initlock(struct fm10k_hw *hw)
 114 {
 115         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 116 }
 117
 118 static void
 119 fm10k_mbx_lock(struct fm10k_hw *hw)
 120 {
 121         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
 122                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
 123 }
 124
 125 static void
 126 fm10k_mbx_unlock(struct fm10k_hw *hw)
 127 {
 128         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
 129 }
 130
 131 /* Stubs needed for linkage when vPMD is disabled */
 132 int __attribute__((weak))
 133 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
 134 {
 135         return -1;
 136 }
 137
 138 uint16_t __attribute__((weak))
 139 fm10k_recv_pkts_vec(
 140         __rte_unused void *rx_queue,
 141         __rte_unused struct rte_mbuf **rx_pkts,
 142         __rte_unused uint16_t nb_pkts)
 143 {
 144         return 0;
 145 }
 146
 147 uint16_t __attribute__((weak))
 148 fm10k_recv_scattered_pkts_vec(
 149                 __rte_unused void *rx_queue,
 150                 __rte_unused struct rte_mbuf **rx_pkts,
 151                 __rte_unused uint16_t nb_pkts)
 152 {
 153         return 0;
 154 }
 155
 156 int __attribute__((weak))
 157 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
 158
 159 {
 160         return -1;
 161 }
 162
 163 void __attribute__((weak))
 164 fm10k_rx_queue_release_mbufs_vec(
 165                 __rte_unused struct fm10k_rx_queue *rxq)
 166 {
 167         return;
 168 }
 169
 170 void __attribute__((weak))
 171 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
 172 {
 173         return;
 174 }
 175
 176 int __attribute__((weak))
 177 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
 178 {
 179         return -1;
 180 }
 181
 182 uint16_t __attribute__((weak))
 183 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue,
 184                            __rte_unused struct rte_mbuf **tx_pkts,
 185                            __rte_unused uint16_t nb_pkts)
 186 {
 187         return 0;
 188 }
 189
 190 /*
 191  * reset queue to initial state, allocate software buffers used when starting
 192  * device.
 193  * return 0 on success
 194  * return -ENOMEM if buffers cannot be allocated
 195  * return -EINVAL if buffers do not satisfy alignment condition
 196  */
 197 static inline int
 198 rx_queue_reset(struct fm10k_rx_queue *q)
 199 {
 200         static const union fm10k_rx_desc zero = {{0} };
 201         uint64_t dma_addr;
 202         int i, diag;
 203         PMD_INIT_FUNC_TRACE();
 204
 205         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 206         if (diag != 0)
 207                 return -ENOMEM;
 208
 209         for (i = 0; i < q->nb_desc; ++i) {
 210                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 211                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 212                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 213                                                 q->nb_desc);
 214                         return -EINVAL;
 215                 }
 216                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 217                 q->hw_ring[i].q.pkt_addr = dma_addr;
 218                 q->hw_ring[i].q.hdr_addr = dma_addr;
 219         }
 220
 221         /* initialize extra software ring entries. Space for these extra
 222          * entries is always allocated.
 223          */
 224         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 225         for (i = 0; i < q->nb_fake_desc; ++i) {
 226                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 227                 q->hw_ring[q->nb_desc + i] = zero;
 228         }
 229
 230         q->next_dd = 0;
 231         q->next_alloc = 0;
 232         q->next_trigger = q->alloc_thresh - 1;
 233         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 234         q->rxrearm_start = 0;
 235         q->rxrearm_nb = 0;
 236
 237         return 0;
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device.
 243  */
 244 static inline void
 245 rx_queue_clean(struct fm10k_rx_queue *q)
 246 {
 247         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* zero faked descriptors */
 256         for (i = 0; i < q->nb_fake_desc; ++i)
 257                 q->hw_ring[q->nb_desc + i] = zero;
 258
 259         /* vPMD driver has a different way of releasing mbufs. */
 260         if (q->rx_using_sse) {
 261                 fm10k_rx_queue_release_mbufs_vec(q);
 262                 return;
 263         }
 264
 265         /* free software buffers */
 266         for (i = 0; i < q->nb_desc; ++i) {
 267                 if (q->sw_ring[i]) {
 268                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 269                         q->sw_ring[i] = NULL;
 270                 }
 271         }
 272 }
 273
 274 /*
 275  * free all queue memory used when releasing the queue (i.e. configure)
 276  */
 277 static inline void
 278 rx_queue_free(struct fm10k_rx_queue *q)
 279 {
 280         PMD_INIT_FUNC_TRACE();
 281         if (q) {
 282                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 283                 rx_queue_clean(q);
 284                 if (q->sw_ring) {
 285                         rte_free(q->sw_ring);
 286                         q->sw_ring = NULL;
 287                 }
 288                 rte_free(q);
 289                 q = NULL;
 290         }
 291 }
 292
 293 /*
 294  * disable RX queue, wait unitl HW finished necessary flush operation
 295  */
 296 static inline int
 297 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 298 {
 299         uint32_t reg, i;
 300
 301         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 302         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 303                         reg & ~FM10K_RXQCTL_ENABLE);
 304
 305         /* Wait 100us at most */
 306         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 307                 rte_delay_us(1);
 308                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 309                 if (!(reg & FM10K_RXQCTL_ENABLE))
 310                         break;
 311         }
 312
 313         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 314                 return -1;
 315
 316         return 0;
 317 }
 318
 319 /*
 320  * reset queue to initial state, allocate software buffers used when starting
 321  * device
 322  */
 323 static inline void
 324 tx_queue_reset(struct fm10k_tx_queue *q)
 325 {
 326         PMD_INIT_FUNC_TRACE();
 327         q->last_free = 0;
 328         q->next_free = 0;
 329         q->nb_used = 0;
 330         q->nb_free = q->nb_desc - 1;
 331         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 332         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 333 }
 334
 335 /*
 336  * clean queue, descriptor rings, free software buffers used when stopping
 337  * device
 338  */
 339 static inline void
 340 tx_queue_clean(struct fm10k_tx_queue *q)
 341 {
 342         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 343         uint32_t i;
 344         PMD_INIT_FUNC_TRACE();
 345
 346         /* zero descriptor rings */
 347         for (i = 0; i < q->nb_desc; ++i)
 348                 q->hw_ring[i] = zero;
 349
 350         /* free software buffers */
 351         for (i = 0; i < q->nb_desc; ++i) {
 352                 if (q->sw_ring[i]) {
 353                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 354                         q->sw_ring[i] = NULL;
 355                 }
 356         }
 357 }
 358
 359 /*
 360  * free all queue memory used when releasing the queue (i.e. configure)
 361  */
 362 static inline void
 363 tx_queue_free(struct fm10k_tx_queue *q)
 364 {
 365         PMD_INIT_FUNC_TRACE();
 366         if (q) {
 367                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 368                 tx_queue_clean(q);
 369                 if (q->rs_tracker.list) {
 370                         rte_free(q->rs_tracker.list);
 371                         q->rs_tracker.list = NULL;
 372                 }
 373                 if (q->sw_ring) {
 374                         rte_free(q->sw_ring);
 375                         q->sw_ring = NULL;
 376                 }
 377                 rte_free(q);
 378                 q = NULL;
 379         }
 380 }
 381
 382 /*
 383  * disable TX queue, wait unitl HW finished necessary flush operation
 384  */
 385 static inline int
 386 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 387 {
 388         uint32_t reg, i;
 389
 390         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 391         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 392                         reg & ~FM10K_TXDCTL_ENABLE);
 393
 394         /* Wait 100us at most */
 395         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 396                 rte_delay_us(1);
 397                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 398                 if (!(reg & FM10K_TXDCTL_ENABLE))
 399                         break;
 400         }
 401
 402         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static int
 409 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 410 {
 411         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 413         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 414         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 415
 416         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 417
 418         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 419                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 420                 return -EINVAL;
 421         }
 422
 423         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 424                 return 0;
 425
 426         if (hw->mac.type == fm10k_mac_vf) {
 427                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 428                 return -EINVAL;
 429         }
 430
 431         /* Check VMDQ queue pool number */
 432         if (vmdq_conf->nb_queue_pools >
 433                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 434                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 435                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 436                         vmdq_conf->nb_queue_pools);
 437                 return -EINVAL;
 438         }
 439
 440         return 0;
 441 }
 442
 443 static const struct fm10k_txq_ops def_txq_ops = {
 444         .reset = tx_queue_reset,
 445 };
 446
 447 static int
 448 fm10k_dev_configure(struct rte_eth_dev *dev)
 449 {
 450         int ret;
 451         struct rte_eth_dev_info dev_info;
 452         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
 453         uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
 454
 455         PMD_INIT_FUNC_TRACE();
 456
 457         if ((rx_offloads & DEV_RX_OFFLOAD_CRC_STRIP) == 0)
 458                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 459
 460         fm10k_dev_infos_get(dev, &dev_info);
 461         if ((rx_offloads & dev_info.rx_offload_capa) != rx_offloads) {
 462                 PMD_DRV_LOG(ERR, "Some Rx offloads are not supported "
 463                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 464                             rx_offloads, dev_info.rx_offload_capa);
 465                 return -ENOTSUP;
 466         }
 467         if ((tx_offloads & dev_info.tx_offload_capa) != tx_offloads) {
 468                 PMD_DRV_LOG(ERR, "Some Tx offloads are not supported "
 469                             "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 470                             tx_offloads, dev_info.tx_offload_capa);
 471                 return -ENOTSUP;
 472         }
 473
 474         /* multipe queue mode checking */
 475         ret  = fm10k_check_mq_mode(dev);
 476         if (ret != 0) {
 477                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 478                             ret);
 479                 return ret;
 480         }
 481
 482         dev->data->scattered_rx = 0;
 483
 484         return 0;
 485 }
 486
 487 /* fls = find last set bit = 32 minus the number of leading zeros */
 488 #ifndef fls
 489 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 490 #endif
 491
 492 static void
 493 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 494 {
 495         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 496         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 497         uint32_t i;
 498
 499         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 500
 501         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 502                 if (!vmdq_conf->pool_map[i].pools)
 503                         continue;
 504                 fm10k_mbx_lock(hw);
 505                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 506                 fm10k_mbx_unlock(hw);
 507         }
 508 }
 509
 510 static void
 511 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 512 {
 513         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 514
 515         /* Add default mac address */
 516         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 517                 MAIN_VSI_POOL_NUMBER);
 518 }
 519
 520 static void
 521 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 522 {
 523         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 524         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 525         uint32_t mrqc, *key, i, reta, j;
 526         uint64_t hf;
 527
 528 #define RSS_KEY_SIZE 40
 529         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 530                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 531                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 532                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 533                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 534                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 535         };
 536
 537         if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 538                 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) {
 539                 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0);
 540                 return;
 541         }
 542
 543         /* random key is rss_intel_key (default) or user provided (rss_key) */
 544         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 545                 key = (uint32_t *)rss_intel_key;
 546         else
 547                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 548
 549         /* Now fill our hash function seeds, 4 bytes at a time */
 550         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 551                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 552
 553         /*
 554          * Fill in redirection table
 555          * The byte-swap is needed because NIC registers are in
 556          * little-endian order.
 557          */
 558         reta = 0;
 559         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 560                 if (j == dev->data->nb_rx_queues)
 561                         j = 0;
 562                 reta = (reta << CHAR_BIT) | j;
 563                 if ((i & 3) == 3)
 564                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 565                                         rte_bswap32(reta));
 566         }
 567
 568         /*
 569          * Generate RSS hash based on packet types, TCP/UDP
 570          * port numbers and/or IPv4/v6 src and dst addresses
 571          */
 572         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 573         mrqc = 0;
 574         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 575         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 576         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 577         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 578         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 579         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 580         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 581         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 582         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 583
 584         if (mrqc == 0) {
 585                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 586                         "supported", hf);
 587                 return;
 588         }
 589
 590         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 591 }
 592
 593 static void
 594 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
 595 {
 596         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 597         uint32_t i;
 598
 599         for (i = 0; i < nb_lport_new; i++) {
 600                 /* Set unicast mode by default. App can change
 601                  * to other mode in other API func.
 602                  */
 603                 fm10k_mbx_lock(hw);
 604                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 605                         FM10K_XCAST_MODE_NONE);
 606                 fm10k_mbx_unlock(hw);
 607         }
 608 }
 609
 610 static void
 611 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 612 {
 613         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 614         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 615         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 616         struct fm10k_macvlan_filter_info *macvlan;
 617         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 618         uint16_t nb_lport_new;
 619
 620         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 621         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 622
 623         fm10k_dev_rss_configure(dev);
 624
 625         /* only PF supports VMDQ */
 626         if (hw->mac.type != fm10k_mac_pf)
 627                 return;
 628
 629         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 630                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 631
 632         /* no pool number change, no need to update logic port and VLAN/MAC */
 633         if (macvlan->nb_queue_pools == nb_queue_pools)
 634                 return;
 635
 636         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 637         fm10k_dev_logic_port_update(dev, nb_lport_new);
 638
 639         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 640         memset(dev->data->mac_addrs, 0,
 641                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 642         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 643                 &dev->data->mac_addrs[0]);
 644         memset(macvlan, 0, sizeof(*macvlan));
 645         macvlan->nb_queue_pools = nb_queue_pools;
 646
 647         if (nb_queue_pools)
 648                 fm10k_dev_vmdq_rx_configure(dev);
 649         else
 650                 fm10k_dev_pf_main_vsi_reset(dev);
 651 }
 652
 653 static int
 654 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 655 {
 656         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 657         int i, ret;
 658         struct fm10k_tx_queue *txq;
 659         uint64_t base_addr;
 660         uint32_t size;
 661
 662         /* Disable TXINT to avoid possible interrupt */
 663         for (i = 0; i < hw->mac.max_queues; i++)
 664                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 665                                 3 << FM10K_TXINT_TIMER_SHIFT);
 666
 667         /* Setup TX queue */
 668         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 669                 txq = dev->data->tx_queues[i];
 670                 base_addr = txq->hw_ring_phys_addr;
 671                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 672
 673                 /* disable queue to avoid issues while updating state */
 674                 ret = tx_queue_disable(hw, i);
 675                 if (ret) {
 676                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 677                         return -1;
 678                 }
 679                 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
 680                  * register is read-only for VF.
 681                  */
 682                 if (fm10k_check_ftag(dev->device->devargs)) {
 683                         if (hw->mac.type == fm10k_mac_pf) {
 684                                 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
 685                                                 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 686                                 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
 687                         } else {
 688                                 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
 689                                 return -ENOTSUP;
 690                         }
 691                 }
 692
 693                 /* set location and size for descriptor ring */
 694                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 695                                 base_addr & UINT64_LOWER_32BITS_MASK);
 696                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 697                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 698                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 699
 700                 /* assign default SGLORT for each TX queue by PF */
 701                 if (hw->mac.type == fm10k_mac_pf)
 702                         FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
 703         }
 704
 705         /* set up vector or scalar TX function as appropriate */
 706         fm10k_set_tx_function(dev);
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         struct fm10k_macvlan_filter_info *macvlan;
 716         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
 717         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
 718         int i, ret;
 719         struct fm10k_rx_queue *rxq;
 720         uint64_t base_addr;
 721         uint32_t size;
 722         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 723         uint32_t logic_port = hw->mac.dglort_map;
 724         uint16_t buf_size;
 725         uint16_t queue_stride = 0;
 726
 727         /* enable RXINT for interrupt mode */
 728         i = 0;
 729         if (rte_intr_dp_is_en(intr_handle)) {
 730                 for (; i < dev->data->nb_rx_queues; i++) {
 731                         FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i));
 732                         if (hw->mac.type == fm10k_mac_pf)
 733                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
 734                                         FM10K_ITR_AUTOMASK |
 735                                         FM10K_ITR_MASK_CLEAR);
 736                         else
 737                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
 738                                         FM10K_ITR_AUTOMASK |
 739                                         FM10K_ITR_MASK_CLEAR);
 740                 }
 741         }
 742         /* Disable other RXINT to avoid possible interrupt */
 743         for (; i < hw->mac.max_queues; i++)
 744                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 745                         3 << FM10K_RXINT_TIMER_SHIFT);
 746
 747         /* Setup RX queues */
 748         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 749                 rxq = dev->data->rx_queues[i];
 750                 base_addr = rxq->hw_ring_phys_addr;
 751                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 752
 753                 /* disable queue to avoid issues while updating state */
 754                 ret = rx_queue_disable(hw, i);
 755                 if (ret) {
 756                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 757                         return -1;
 758                 }
 759
 760                 /* Setup the Base and Length of the Rx Descriptor Ring */
 761                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 762                                 base_addr & UINT64_LOWER_32BITS_MASK);
 763                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 764                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 765                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 766
 767                 /* Configure the Rx buffer size for one buff without split */
 768                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 769                         RTE_PKTMBUF_HEADROOM);
 770                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 771                  * reserved for this purpose, and the worst case could be 511B.
 772                  * But SRR reg assumes all buffers have the same size. In order
 773                  * to fill the gap, we'll have to consider the worst case and
 774                  * assume 512B is reserved. If we don't do so, it's possible
 775                  * for HW to overwrite data to next mbuf.
 776                  */
 777                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 778
 779                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 780                                 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
 781                                 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
 782
 783                 /* It adds dual VLAN length for supporting dual VLAN */
 784                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 785                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 786                         rxq->offloads & DEV_RX_OFFLOAD_SCATTER) {
 787                         uint32_t reg;
 788                         dev->data->scattered_rx = 1;
 789                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 790                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 791                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 792                 }
 793
 794                 /* Enable drop on empty, it's RO for VF */
 795                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 796                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 797
 798                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 799                 FM10K_WRITE_FLUSH(hw);
 800         }
 801
 802         /* Configure VMDQ/RSS if applicable */
 803         fm10k_dev_mq_rx_configure(dev);
 804
 805         /* Decide the best RX function */
 806         fm10k_set_rx_function(dev);
 807
 808         /* update RX_SGLORT for loopback suppress*/
 809         if (hw->mac.type != fm10k_mac_pf)
 810                 return 0;
 811         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 812         if (macvlan->nb_queue_pools)
 813                 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
 814         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 815                 if (i && queue_stride && !(i % queue_stride))
 816                         logic_port++;
 817                 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
 818         }
 819
 820         return 0;
 821 }
 822
 823 static int
 824 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 825 {
 826         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 827         int err = -1;
 828         uint32_t reg;
 829         struct fm10k_rx_queue *rxq;
 830
 831         PMD_INIT_FUNC_TRACE();
 832
 833         if (rx_queue_id < dev->data->nb_rx_queues) {
 834                 rxq = dev->data->rx_queues[rx_queue_id];
 835                 err = rx_queue_reset(rxq);
 836                 if (err == -ENOMEM) {
 837                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 838                         return err;
 839                 } else if (err == -EINVAL) {
 840                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 841                                 " %d", err);
 842                         return err;
 843                 }
 844
 845                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 846                  * Note: this must be done AFTER the queue is enabled on real
 847                  * hardware, but BEFORE the queue is enabled when using the
 848                  * emulation platform. Do it in both places for now and remove
 849                  * this comment and the following two register writes when the
 850                  * emulation platform is no longer being used.
 851                  */
 852                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 853                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 854
 855                 /* Set PF ownership flag for PF devices */
 856                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 857                 if (hw->mac.type == fm10k_mac_pf)
 858                         reg |= FM10K_RXQCTL_PF;
 859                 reg |= FM10K_RXQCTL_ENABLE;
 860                 /* enable RX queue */
 861                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 862                 FM10K_WRITE_FLUSH(hw);
 863
 864                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 865                  * Note: this must be done AFTER the queue is enabled
 866                  */
 867                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 868                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 869                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 870         }
 871
 872         return err;
 873 }
 874
 875 static int
 876 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         if (rx_queue_id < dev->data->nb_rx_queues) {
 883                 /* Disable RX queue */
 884                 rx_queue_disable(hw, rx_queue_id);
 885
 886                 /* Free mbuf and clean HW ring */
 887                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 888                 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 889         }
 890
 891         return 0;
 892 }
 893
 894 static int
 895 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 896 {
 897         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 898         /** @todo - this should be defined in the shared code */
 899 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 900         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 901         int err = 0;
 902
 903         PMD_INIT_FUNC_TRACE();
 904
 905         if (tx_queue_id < dev->data->nb_tx_queues) {
 906                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 907
 908                 q->ops->reset(q);
 909
 910                 /* reset head and tail pointers */
 911                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 912                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 913
 914                 /* enable TX queue */
 915                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 916                                         FM10K_TXDCTL_ENABLE | txdctl);
 917                 FM10K_WRITE_FLUSH(hw);
 918                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 919         } else
 920                 err = -1;
 921
 922         return err;
 923 }
 924
 925 static int
 926 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 927 {
 928         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         if (tx_queue_id < dev->data->nb_tx_queues) {
 933                 tx_queue_disable(hw, tx_queue_id);
 934                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 935                 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 936         }
 937
 938         return 0;
 939 }
 940
 941 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 942 {
 943         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 944                 != FM10K_DGLORTMAP_NONE);
 945 }
 946
 947 static void
 948 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 949 {
 950         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         int status;
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         /* Return if it didn't acquire valid glort range */
 956         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 957                 return;
 958
 959         fm10k_mbx_lock(hw);
 960         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 961                                 FM10K_XCAST_MODE_PROMISC);
 962         fm10k_mbx_unlock(hw);
 963
 964         if (status != FM10K_SUCCESS)
 965                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 966 }
 967
 968 static void
 969 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 970 {
 971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 972         uint8_t mode;
 973         int status;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         /* Return if it didn't acquire valid glort range */
 978         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
 979                 return;
 980
 981         if (dev->data->all_multicast == 1)
 982                 mode = FM10K_XCAST_MODE_ALLMULTI;
 983         else
 984                 mode = FM10K_XCAST_MODE_NONE;
 985
 986         fm10k_mbx_lock(hw);
 987         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 988                                 mode);
 989         fm10k_mbx_unlock(hw);
 990
 991         if (status != FM10K_SUCCESS)
 992                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 993 }
 994
 995 static void
 996 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 997 {
 998         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 999         int status;
1000
1001         PMD_INIT_FUNC_TRACE();
1002
1003         /* Return if it didn't acquire valid glort range */
1004         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1005                 return;
1006
1007         /* If promiscuous mode is enabled, it doesn't make sense to enable
1008          * allmulticast and disable promiscuous since fm10k only can select
1009          * one of the modes.
1010          */
1011         if (dev->data->promiscuous) {
1012                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1013                         "needn't enable allmulticast");
1014                 return;
1015         }
1016
1017         fm10k_mbx_lock(hw);
1018         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1019                                 FM10K_XCAST_MODE_ALLMULTI);
1020         fm10k_mbx_unlock(hw);
1021
1022         if (status != FM10K_SUCCESS)
1023                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1024 }
1025
1026 static void
1027 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1028 {
1029         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1030         int status;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Return if it didn't acquire valid glort range */
1035         if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw))
1036                 return;
1037
1038         if (dev->data->promiscuous) {
1039                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1040                         "since promisc mode is enabled");
1041                 return;
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         /* Change mode to unicast mode */
1046         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1047                                 FM10K_XCAST_MODE_NONE);
1048         fm10k_mbx_unlock(hw);
1049
1050         if (status != FM10K_SUCCESS)
1051                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1052 }
1053
1054 static void
1055 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1056 {
1057         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1058         uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1059         uint16_t nb_queue_pools;
1060         struct fm10k_macvlan_filter_info *macvlan;
1061
1062         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1063         nb_queue_pools = macvlan->nb_queue_pools;
1064         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1065         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1066
1067         /* GLORT 0x0-0x3F are used by PF and VMDQ,  0x40-0x7F used by FD */
1068         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1069         dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070                         hw->mac.dglort_map;
1071         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1072         /* Configure VMDQ/RSS DGlort Decoder */
1073         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1074
1075         /* Flow Director configurations, only queue number is valid. */
1076         dglortdec = fls(dev->data->nb_rx_queues - 1);
1077         dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1078                         (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1079         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1080         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1081
1082         /* Invalidate all other GLORT entries */
1083         for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1084                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1085                                 FM10K_DGLORTMAP_NONE);
1086 }
1087
1088 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1089 static int
1090 fm10k_dev_start(struct rte_eth_dev *dev)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093         int i, diag;
1094
1095         PMD_INIT_FUNC_TRACE();
1096
1097         /* stop, init, then start the hw */
1098         diag = fm10k_stop_hw(hw);
1099         if (diag != FM10K_SUCCESS) {
1100                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1101                 return -EIO;
1102         }
1103
1104         diag = fm10k_init_hw(hw);
1105         if (diag != FM10K_SUCCESS) {
1106                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1107                 return -EIO;
1108         }
1109
1110         diag = fm10k_start_hw(hw);
1111         if (diag != FM10K_SUCCESS) {
1112                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1113                 return -EIO;
1114         }
1115
1116         diag = fm10k_dev_tx_init(dev);
1117         if (diag) {
1118                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1119                 return diag;
1120         }
1121
1122         if (fm10k_dev_rxq_interrupt_setup(dev))
1123                 return -EIO;
1124
1125         diag = fm10k_dev_rx_init(dev);
1126         if (diag) {
1127                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1128                 return diag;
1129         }
1130
1131         if (hw->mac.type == fm10k_mac_pf)
1132                 fm10k_dev_dglort_map_configure(dev);
1133
1134         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1135                 struct fm10k_rx_queue *rxq;
1136                 rxq = dev->data->rx_queues[i];
1137
1138                 if (rxq->rx_deferred_start)
1139                         continue;
1140                 diag = fm10k_dev_rx_queue_start(dev, i);
1141                 if (diag != 0) {
1142                         int j;
1143                         for (j = 0; j < i; ++j)
1144                                 rx_queue_clean(dev->data->rx_queues[j]);
1145                         return diag;
1146                 }
1147         }
1148
1149         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1150                 struct fm10k_tx_queue *txq;
1151                 txq = dev->data->tx_queues[i];
1152
1153                 if (txq->tx_deferred_start)
1154                         continue;
1155                 diag = fm10k_dev_tx_queue_start(dev, i);
1156                 if (diag != 0) {
1157                         int j;
1158                         for (j = 0; j < i; ++j)
1159                                 tx_queue_clean(dev->data->tx_queues[j]);
1160                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
1161                                 rx_queue_clean(dev->data->rx_queues[j]);
1162                         return diag;
1163                 }
1164         }
1165
1166         /* Update default vlan when not in VMDQ mode */
1167         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1168                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1169
1170         fm10k_link_update(dev, 0);
1171
1172         return 0;
1173 }
1174
1175 static void
1176 fm10k_dev_stop(struct rte_eth_dev *dev)
1177 {
1178         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1179         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1180         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
1181         int i;
1182
1183         PMD_INIT_FUNC_TRACE();
1184
1185         if (dev->data->tx_queues)
1186                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1187                         fm10k_dev_tx_queue_stop(dev, i);
1188
1189         if (dev->data->rx_queues)
1190                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1191                         fm10k_dev_rx_queue_stop(dev, i);
1192
1193         /* Disable datapath event */
1194         if (rte_intr_dp_is_en(intr_handle)) {
1195                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1196                         FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1197                                 3 << FM10K_RXINT_TIMER_SHIFT);
1198                         if (hw->mac.type == fm10k_mac_pf)
1199                                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)),
1200                                         FM10K_ITR_MASK_SET);
1201                         else
1202                                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)),
1203                                         FM10K_ITR_MASK_SET);
1204                 }
1205         }
1206         /* Clean datapath event and queue/vec mapping */
1207         rte_intr_efd_disable(intr_handle);
1208         rte_free(intr_handle->intr_vec);
1209         intr_handle->intr_vec = NULL;
1210 }
1211
1212 static void
1213 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1214 {
1215         int i;
1216
1217         PMD_INIT_FUNC_TRACE();
1218
1219         if (dev->data->tx_queues) {
1220                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1221                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1222
1223                         tx_queue_free(txq);
1224                 }
1225         }
1226
1227         if (dev->data->rx_queues) {
1228                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1229                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1230         }
1231 }
1232
1233 static void
1234 fm10k_dev_close(struct rte_eth_dev *dev)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237
1238         PMD_INIT_FUNC_TRACE();
1239
1240         fm10k_mbx_lock(hw);
1241         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1242                 MAX_LPORT_NUM, false);
1243         fm10k_mbx_unlock(hw);
1244
1245         /* allow 100ms for device to quiesce */
1246         rte_delay_us(FM10K_SWITCH_QUIESCE_US);
1247
1248         /* Stop mailbox service first */
1249         fm10k_close_mbx_service(hw);
1250         fm10k_dev_stop(dev);
1251         fm10k_dev_queue_release(dev);
1252         fm10k_stop_hw(hw);
1253 }
1254
1255 static int
1256 fm10k_link_update(struct rte_eth_dev *dev,
1257         __rte_unused int wait_to_complete)
1258 {
1259         struct fm10k_dev_info *dev_info =
1260                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1261         PMD_INIT_FUNC_TRACE();
1262
1263         dev->data->dev_link.link_speed  = ETH_SPEED_NUM_50G;
1264         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1265         dev->data->dev_link.link_status =
1266                 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP;
1267         dev->data->dev_link.link_autoneg = ETH_LINK_FIXED;
1268
1269         return 0;
1270 }
1271
1272 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1273         struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
1274 {
1275         unsigned i, q;
1276         unsigned count = 0;
1277
1278         if (xstats_names != NULL) {
1279                 /* Note: limit checked in rte_eth_xstats_names() */
1280
1281                 /* Global stats */
1282                 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1283                         snprintf(xstats_names[count].name,
1284                                 sizeof(xstats_names[count].name),
1285                                 "%s", fm10k_hw_stats_strings[count].name);
1286                         count++;
1287                 }
1288
1289                 /* PF queue stats */
1290                 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1291                         for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1292                                 snprintf(xstats_names[count].name,
1293                                         sizeof(xstats_names[count].name),
1294                                         "rx_q%u_%s", q,
1295                                         fm10k_hw_stats_rx_q_strings[i].name);
1296                                 count++;
1297                         }
1298                         for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1299                                 snprintf(xstats_names[count].name,
1300                                         sizeof(xstats_names[count].name),
1301                                         "tx_q%u_%s", q,
1302                                         fm10k_hw_stats_tx_q_strings[i].name);
1303                                 count++;
1304                         }
1305                 }
1306         }
1307         return FM10K_NB_XSTATS;
1308 }
1309
1310 static int
1311 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1312                  unsigned n)
1313 {
1314         struct fm10k_hw_stats *hw_stats =
1315                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1316         unsigned i, q, count = 0;
1317
1318         if (n < FM10K_NB_XSTATS)
1319                 return FM10K_NB_XSTATS;
1320
1321         /* Global stats */
1322         for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1323                 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1324                         fm10k_hw_stats_strings[count].offset);
1325                 xstats[count].id = count;
1326                 count++;
1327         }
1328
1329         /* PF queue stats */
1330         for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1331                 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1332                         xstats[count].value =
1333                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1334                                 fm10k_hw_stats_rx_q_strings[i].offset);
1335                         xstats[count].id = count;
1336                         count++;
1337                 }
1338                 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1339                         xstats[count].value =
1340                                 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1341                                 fm10k_hw_stats_tx_q_strings[i].offset);
1342                         xstats[count].id = count;
1343                         count++;
1344                 }
1345         }
1346
1347         return FM10K_NB_XSTATS;
1348 }
1349
1350 static int
1351 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1352 {
1353         uint64_t ipackets, opackets, ibytes, obytes;
1354         struct fm10k_hw *hw =
1355                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1356         struct fm10k_hw_stats *hw_stats =
1357                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1358         int i;
1359
1360         PMD_INIT_FUNC_TRACE();
1361
1362         fm10k_update_hw_stats(hw, hw_stats);
1363
1364         ipackets = opackets = ibytes = obytes = 0;
1365         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1366                 (i < hw->mac.max_queues); ++i) {
1367                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1368                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1369                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1370                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1371                 ipackets += stats->q_ipackets[i];
1372                 opackets += stats->q_opackets[i];
1373                 ibytes   += stats->q_ibytes[i];
1374                 obytes   += stats->q_obytes[i];
1375         }
1376         stats->ipackets = ipackets;
1377         stats->opackets = opackets;
1378         stats->ibytes = ibytes;
1379         stats->obytes = obytes;
1380         return 0;
1381 }
1382
1383 static void
1384 fm10k_stats_reset(struct rte_eth_dev *dev)
1385 {
1386         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1387         struct fm10k_hw_stats *hw_stats =
1388                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1389
1390         PMD_INIT_FUNC_TRACE();
1391
1392         memset(hw_stats, 0, sizeof(*hw_stats));
1393         fm10k_rebind_hw_stats(hw, hw_stats);
1394 }
1395
1396 static void
1397 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1398         struct rte_eth_dev_info *dev_info)
1399 {
1400         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1401         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
1402
1403         PMD_INIT_FUNC_TRACE();
1404
1405         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1406         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1407         dev_info->max_rx_queues      = hw->mac.max_queues;
1408         dev_info->max_tx_queues      = hw->mac.max_queues;
1409         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1410         dev_info->max_hash_mac_addrs = 0;
1411         dev_info->max_vfs            = pdev->max_vfs;
1412         dev_info->vmdq_pool_base     = 0;
1413         dev_info->vmdq_queue_base    = 0;
1414         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1415         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1416         dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev);
1417         dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) |
1418                                     dev_info->rx_queue_offload_capa;
1419         dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev);
1420         dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) |
1421                                     dev_info->tx_queue_offload_capa;
1422
1423         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1424         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1425
1426         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1427                 .rx_thresh = {
1428                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1429                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1430                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1431                 },
1432                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1433                 .rx_drop_en = 0,
1434                 .offloads = 0,
1435         };
1436
1437         dev_info->default_txconf = (struct rte_eth_txconf) {
1438                 .tx_thresh = {
1439                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1440                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1441                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1442                 },
1443                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1444                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1445                 .offloads = 0,
1446         };
1447
1448         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1449                 .nb_max = FM10K_MAX_RX_DESC,
1450                 .nb_min = FM10K_MIN_RX_DESC,
1451                 .nb_align = FM10K_MULT_RX_DESC,
1452         };
1453
1454         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1455                 .nb_max = FM10K_MAX_TX_DESC,
1456                 .nb_min = FM10K_MIN_TX_DESC,
1457                 .nb_align = FM10K_MULT_TX_DESC,
1458                 .nb_seg_max = FM10K_TX_MAX_SEG,
1459                 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG,
1460         };
1461
1462         dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
1463                         ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G |
1464                         ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G;
1465 }
1466
1467 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
1468 static const uint32_t *
1469 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1470 {
1471         if (dev->rx_pkt_burst == fm10k_recv_pkts ||
1472             dev->rx_pkt_burst == fm10k_recv_scattered_pkts) {
1473                 static uint32_t ptypes[] = {
1474                         /* refers to rx_desc_to_ol_flags() */
1475                         RTE_PTYPE_L2_ETHER,
1476                         RTE_PTYPE_L3_IPV4,
1477                         RTE_PTYPE_L3_IPV4_EXT,
1478                         RTE_PTYPE_L3_IPV6,
1479                         RTE_PTYPE_L3_IPV6_EXT,
1480                         RTE_PTYPE_L4_TCP,
1481                         RTE_PTYPE_L4_UDP,
1482                         RTE_PTYPE_UNKNOWN
1483                 };
1484
1485                 return ptypes;
1486         } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec ||
1487                    dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) {
1488                 static uint32_t ptypes_vec[] = {
1489                         /* refers to fm10k_desc_to_pktype_v() */
1490                         RTE_PTYPE_L3_IPV4,
1491                         RTE_PTYPE_L3_IPV4_EXT,
1492                         RTE_PTYPE_L3_IPV6,
1493                         RTE_PTYPE_L3_IPV6_EXT,
1494                         RTE_PTYPE_L4_TCP,
1495                         RTE_PTYPE_L4_UDP,
1496                         RTE_PTYPE_TUNNEL_GENEVE,
1497                         RTE_PTYPE_TUNNEL_NVGRE,
1498                         RTE_PTYPE_TUNNEL_VXLAN,
1499                         RTE_PTYPE_TUNNEL_GRE,
1500                         RTE_PTYPE_UNKNOWN
1501                 };
1502
1503                 return ptypes_vec;
1504         }
1505
1506         return NULL;
1507 }
1508 #else
1509 static const uint32_t *
1510 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1511 {
1512         return NULL;
1513 }
1514 #endif
1515
1516 static int
1517 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1518 {
1519         s32 result;
1520         uint16_t mac_num = 0;
1521         uint32_t vid_idx, vid_bit, mac_index;
1522         struct fm10k_hw *hw;
1523         struct fm10k_macvlan_filter_info *macvlan;
1524         struct rte_eth_dev_data *data = dev->data;
1525
1526         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1527         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1528
1529         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1530                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1531                 return -EINVAL;
1532         }
1533
1534         if (vlan_id > ETH_VLAN_ID_MAX) {
1535                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1536                 return -EINVAL;
1537         }
1538
1539         vid_idx = FM10K_VFTA_IDX(vlan_id);
1540         vid_bit = FM10K_VFTA_BIT(vlan_id);
1541         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1542         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1543                 return 0;
1544         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1545         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1546                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1547                         "in the VLAN filter table");
1548                 return -EINVAL;
1549         }
1550
1551         fm10k_mbx_lock(hw);
1552         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1553         fm10k_mbx_unlock(hw);
1554         if (result != FM10K_SUCCESS) {
1555                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1556                 return -EIO;
1557         }
1558
1559         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1560                         (result == FM10K_SUCCESS); mac_index++) {
1561                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1562                         continue;
1563                 if (mac_num > macvlan->mac_num - 1) {
1564                         PMD_INIT_LOG(ERR, "MAC address number "
1565                                         "not match");
1566                         break;
1567                 }
1568                 fm10k_mbx_lock(hw);
1569                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1570                         data->mac_addrs[mac_index].addr_bytes,
1571                         vlan_id, on, 0);
1572                 fm10k_mbx_unlock(hw);
1573                 mac_num++;
1574         }
1575         if (result != FM10K_SUCCESS) {
1576                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1577                 return -EIO;
1578         }
1579
1580         if (on) {
1581                 macvlan->vlan_num++;
1582                 macvlan->vfta[vid_idx] |= vid_bit;
1583         } else {
1584                 macvlan->vlan_num--;
1585                 macvlan->vfta[vid_idx] &= ~vid_bit;
1586         }
1587         return 0;
1588 }
1589
1590 static int
1591 fm10k_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1592 {
1593         if (mask & ETH_VLAN_STRIP_MASK) {
1594                 if (!(dev->data->dev_conf.rxmode.offloads &
1595                         DEV_RX_OFFLOAD_VLAN_STRIP))
1596                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1597                                         "always on in fm10k");
1598         }
1599
1600         if (mask & ETH_VLAN_EXTEND_MASK) {
1601                 if (dev->data->dev_conf.rxmode.offloads &
1602                         DEV_RX_OFFLOAD_VLAN_EXTEND)
1603                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1604                                         "supported in fm10k");
1605         }
1606
1607         if (mask & ETH_VLAN_FILTER_MASK) {
1608                 if (!(dev->data->dev_conf.rxmode.offloads &
1609                         DEV_RX_OFFLOAD_VLAN_FILTER))
1610                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1611         }
1612
1613         return 0;
1614 }
1615
1616 /* Add/Remove a MAC address, and update filters to main VSI */
1617 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1618                 const u8 *mac, bool add, uint32_t pool)
1619 {
1620         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1621         struct fm10k_macvlan_filter_info *macvlan;
1622         uint32_t i, j, k;
1623
1624         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1625
1626         if (pool != MAIN_VSI_POOL_NUMBER) {
1627                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1628                         "mac to pool %u", pool);
1629                 return;
1630         }
1631         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1632                 if (!macvlan->vfta[j])
1633                         continue;
1634                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1635                         if (!(macvlan->vfta[j] & (1 << k)))
1636                                 continue;
1637                         if (i + 1 > macvlan->vlan_num) {
1638                                 PMD_INIT_LOG(ERR, "vlan number not match");
1639                                 return;
1640                         }
1641                         fm10k_mbx_lock(hw);
1642                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1643                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1644                         fm10k_mbx_unlock(hw);
1645                         i++;
1646                 }
1647         }
1648 }
1649
1650 /* Add/Remove a MAC address, and update filters to VMDQ */
1651 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1652                 const u8 *mac, bool add, uint32_t pool)
1653 {
1654         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1655         struct fm10k_macvlan_filter_info *macvlan;
1656         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1657         uint32_t i;
1658
1659         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1660         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1661
1662         if (pool > macvlan->nb_queue_pools) {
1663                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1664                         " Max pool is %u",
1665                         pool, macvlan->nb_queue_pools);
1666                 return;
1667         }
1668         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1669                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1670                         continue;
1671                 fm10k_mbx_lock(hw);
1672                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1673                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1674                 fm10k_mbx_unlock(hw);
1675         }
1676 }
1677
1678 /* Add/Remove a MAC address, and update filters */
1679 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1680                 const u8 *mac, bool add, uint32_t pool)
1681 {
1682         struct fm10k_macvlan_filter_info *macvlan;
1683
1684         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1685
1686         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1687                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1688         else
1689                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1690
1691         if (add)
1692                 macvlan->mac_num++;
1693         else
1694                 macvlan->mac_num--;
1695 }
1696
1697 /* Add a MAC address, and update filters */
1698 static int
1699 fm10k_macaddr_add(struct rte_eth_dev *dev,
1700                 struct ether_addr *mac_addr,
1701                 uint32_t index,
1702                 uint32_t pool)
1703 {
1704         struct fm10k_macvlan_filter_info *macvlan;
1705
1706         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1707         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1708         macvlan->mac_vmdq_id[index] = pool;
1709         return 0;
1710 }
1711
1712 /* Remove a MAC address, and update filters */
1713 static void
1714 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1715 {
1716         struct rte_eth_dev_data *data = dev->data;
1717         struct fm10k_macvlan_filter_info *macvlan;
1718
1719         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1720         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1721                         FALSE, macvlan->mac_vmdq_id[index]);
1722         macvlan->mac_vmdq_id[index] = 0;
1723 }
1724
1725 static inline int
1726 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1727 {
1728         if ((request < min) || (request > max) || ((request % mult) != 0))
1729                 return -1;
1730         else
1731                 return 0;
1732 }
1733
1734
1735 static inline int
1736 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1737 {
1738         if ((request < min) || (request > max) || ((div % request) != 0))
1739                 return -1;
1740         else
1741                 return 0;
1742 }
1743
1744 static inline int
1745 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1746 {
1747         uint16_t rx_free_thresh;
1748
1749         if (conf->rx_free_thresh == 0)
1750                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1751         else
1752                 rx_free_thresh = conf->rx_free_thresh;
1753
1754         /* make sure the requested threshold satisfies the constraints */
1755         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1756                         FM10K_RX_FREE_THRESH_MAX(q),
1757                         FM10K_RX_FREE_THRESH_DIV(q),
1758                         rx_free_thresh)) {
1759                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1760                         "less than or equal to %u, "
1761                         "greater than or equal to %u, "
1762                         "and a divisor of %u",
1763                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1764                         FM10K_RX_FREE_THRESH_MIN(q),
1765                         FM10K_RX_FREE_THRESH_DIV(q));
1766                 return -EINVAL;
1767         }
1768
1769         q->alloc_thresh = rx_free_thresh;
1770         q->drop_en = conf->rx_drop_en;
1771         q->rx_deferred_start = conf->rx_deferred_start;
1772
1773         return 0;
1774 }
1775
1776 /*
1777  * Hardware requires specific alignment for Rx packet buffers. At
1778  * least one of the following two conditions must be satisfied.
1779  *  1. Address is 512B aligned
1780  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1781  *
1782  * As such, the driver may need to adjust the DMA address within the
1783  * buffer by up to 512B.
1784  *
1785  * return 1 if the element size is valid, otherwise return 0.
1786  */
1787 static int
1788 mempool_element_size_valid(struct rte_mempool *mp)
1789 {
1790         uint32_t min_size;
1791
1792         /* elt_size includes mbuf header and headroom */
1793         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1794                         RTE_PKTMBUF_HEADROOM;
1795
1796         /* account for up to 512B of alignment */
1797         min_size -= FM10K_RX_DATABUF_ALIGN;
1798
1799         /* sanity check for overflow */
1800         if (min_size > mp->elt_size)
1801                 return 0;
1802
1803         /* size is valid */
1804         return 1;
1805 }
1806
1807 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev)
1808 {
1809         RTE_SET_USED(dev);
1810
1811         return (uint64_t)(DEV_RX_OFFLOAD_SCATTER);
1812 }
1813
1814 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev)
1815 {
1816         RTE_SET_USED(dev);
1817
1818         return  (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP  |
1819                            DEV_RX_OFFLOAD_VLAN_FILTER |
1820                            DEV_RX_OFFLOAD_IPV4_CKSUM  |
1821                            DEV_RX_OFFLOAD_UDP_CKSUM   |
1822                            DEV_RX_OFFLOAD_TCP_CKSUM   |
1823                            DEV_RX_OFFLOAD_JUMBO_FRAME |
1824                            DEV_RX_OFFLOAD_CRC_STRIP   |
1825                            DEV_RX_OFFLOAD_HEADER_SPLIT);
1826 }
1827
1828 static int
1829 fm10k_check_rx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
1830 {
1831         uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
1832         uint64_t queue_supported = fm10k_get_rx_queue_offloads_capa(dev);
1833         uint64_t port_supported = fm10k_get_rx_port_offloads_capa(dev);
1834
1835         if ((requested & (queue_supported | port_supported)) != requested)
1836                 return 0;
1837
1838         if ((port_offloads ^ requested) & port_supported)
1839                 return 0;
1840
1841         return 1;
1842 }
1843
1844 static int
1845 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1846         uint16_t nb_desc, unsigned int socket_id,
1847         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1848 {
1849         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1850         struct fm10k_dev_info *dev_info =
1851                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
1852         struct fm10k_rx_queue *q;
1853         const struct rte_memzone *mz;
1854
1855         PMD_INIT_FUNC_TRACE();
1856
1857         if (!fm10k_check_rx_queue_offloads(dev, conf->offloads)) {
1858                 PMD_INIT_LOG(ERR, "%p: Rx queue offloads 0x%" PRIx64
1859                         " don't match port offloads 0x%" PRIx64
1860                         " or supported port offloads 0x%" PRIx64
1861                         " or supported queue offloads 0x%" PRIx64,
1862                         (void *)dev, conf->offloads,
1863                         dev->data->dev_conf.rxmode.offloads,
1864                         fm10k_get_rx_port_offloads_capa(dev),
1865                         fm10k_get_rx_queue_offloads_capa(dev));
1866                 return -ENOTSUP;
1867         }
1868
1869         /* make sure the mempool element size can account for alignment. */
1870         if (!mempool_element_size_valid(mp)) {
1871                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1872                 return -EINVAL;
1873         }
1874
1875         /* make sure a valid number of descriptors have been requested */
1876         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1877                                 FM10K_MULT_RX_DESC, nb_desc)) {
1878                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1879                         "less than or equal to %"PRIu32", "
1880                         "greater than or equal to %u, "
1881                         "and a multiple of %u",
1882                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1883                         FM10K_MULT_RX_DESC);
1884                 return -EINVAL;
1885         }
1886
1887         /*
1888          * if this queue existed already, free the associated memory. The
1889          * queue cannot be reused in case we need to allocate memory on
1890          * different socket than was previously used.
1891          */
1892         if (dev->data->rx_queues[queue_id] != NULL) {
1893                 rx_queue_free(dev->data->rx_queues[queue_id]);
1894                 dev->data->rx_queues[queue_id] = NULL;
1895         }
1896
1897         /* allocate memory for the queue structure */
1898         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1899                                 socket_id);
1900         if (q == NULL) {
1901                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1902                 return -ENOMEM;
1903         }
1904
1905         /* setup queue */
1906         q->mp = mp;
1907         q->nb_desc = nb_desc;
1908         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1909         q->port_id = dev->data->port_id;
1910         q->queue_id = queue_id;
1911         q->tail_ptr = (volatile uint32_t *)
1912                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1913         q->offloads = conf->offloads;
1914         if (handle_rxconf(q, conf))
1915                 return -EINVAL;
1916
1917         /* allocate memory for the software ring */
1918         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1919                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1920                         RTE_CACHE_LINE_SIZE, socket_id);
1921         if (q->sw_ring == NULL) {
1922                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1923                 rte_free(q);
1924                 return -ENOMEM;
1925         }
1926
1927         /*
1928          * allocate memory for the hardware descriptor ring. A memzone large
1929          * enough to hold the maximum ring size is requested to allow for
1930          * resizing in later calls to the queue setup function.
1931          */
1932         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1933                                       FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1934                                       socket_id);
1935         if (mz == NULL) {
1936                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1937                 rte_free(q->sw_ring);
1938                 rte_free(q);
1939                 return -ENOMEM;
1940         }
1941         q->hw_ring = mz->addr;
1942         q->hw_ring_phys_addr = mz->iova;
1943
1944         /* Check if number of descs satisfied Vector requirement */
1945         if (!rte_is_power_of_2(nb_desc)) {
1946                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1947                                     "preconditions - canceling the feature for "
1948                                     "the whole port[%d]",
1949                              q->queue_id, q->port_id);
1950                 dev_info->rx_vec_allowed = false;
1951         } else
1952                 fm10k_rxq_vec_setup(q);
1953
1954         dev->data->rx_queues[queue_id] = q;
1955         return 0;
1956 }
1957
1958 static void
1959 fm10k_rx_queue_release(void *queue)
1960 {
1961         PMD_INIT_FUNC_TRACE();
1962
1963         rx_queue_free(queue);
1964 }
1965
1966 static inline int
1967 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1968 {
1969         uint16_t tx_free_thresh;
1970         uint16_t tx_rs_thresh;
1971
1972         /* constraint MACROs require that tx_free_thresh is configured
1973          * before tx_rs_thresh */
1974         if (conf->tx_free_thresh == 0)
1975                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1976         else
1977                 tx_free_thresh = conf->tx_free_thresh;
1978
1979         /* make sure the requested threshold satisfies the constraints */
1980         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1981                         FM10K_TX_FREE_THRESH_MAX(q),
1982                         FM10K_TX_FREE_THRESH_DIV(q),
1983                         tx_free_thresh)) {
1984                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1985                         "less than or equal to %u, "
1986                         "greater than or equal to %u, "
1987                         "and a divisor of %u",
1988                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1989                         FM10K_TX_FREE_THRESH_MIN(q),
1990                         FM10K_TX_FREE_THRESH_DIV(q));
1991                 return -EINVAL;
1992         }
1993
1994         q->free_thresh = tx_free_thresh;
1995
1996         if (conf->tx_rs_thresh == 0)
1997                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1998         else
1999                 tx_rs_thresh = conf->tx_rs_thresh;
2000
2001         q->tx_deferred_start = conf->tx_deferred_start;
2002
2003         /* make sure the requested threshold satisfies the constraints */
2004         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
2005                         FM10K_TX_RS_THRESH_MAX(q),
2006                         FM10K_TX_RS_THRESH_DIV(q),
2007                         tx_rs_thresh)) {
2008                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
2009                         "less than or equal to %u, "
2010                         "greater than or equal to %u, "
2011                         "and a divisor of %u",
2012                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
2013                         FM10K_TX_RS_THRESH_MIN(q),
2014                         FM10K_TX_RS_THRESH_DIV(q));
2015                 return -EINVAL;
2016         }
2017
2018         q->rs_thresh = tx_rs_thresh;
2019
2020         return 0;
2021 }
2022
2023 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev)
2024 {
2025         RTE_SET_USED(dev);
2026
2027         return 0;
2028 }
2029
2030 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev)
2031 {
2032         RTE_SET_USED(dev);
2033
2034         return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT |
2035                           DEV_TX_OFFLOAD_IPV4_CKSUM  |
2036                           DEV_TX_OFFLOAD_UDP_CKSUM   |
2037                           DEV_TX_OFFLOAD_TCP_CKSUM   |
2038                           DEV_TX_OFFLOAD_TCP_TSO);
2039 }
2040
2041 static int
2042 fm10k_check_tx_queue_offloads(struct rte_eth_dev *dev, uint64_t requested)
2043 {
2044         uint64_t port_offloads = dev->data->dev_conf.txmode.offloads;
2045         uint64_t queue_supported = fm10k_get_tx_queue_offloads_capa(dev);
2046         uint64_t port_supported = fm10k_get_tx_port_offloads_capa(dev);
2047
2048         if ((requested & (queue_supported | port_supported)) != requested)
2049                 return 0;
2050
2051         if ((port_offloads ^ requested) & port_supported)
2052                 return 0;
2053
2054         return 1;
2055 }
2056
2057 static int
2058 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
2059         uint16_t nb_desc, unsigned int socket_id,
2060         const struct rte_eth_txconf *conf)
2061 {
2062         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2063         struct fm10k_tx_queue *q;
2064         const struct rte_memzone *mz;
2065
2066         PMD_INIT_FUNC_TRACE();
2067
2068         if (!fm10k_check_tx_queue_offloads(dev, conf->offloads)) {
2069                 PMD_INIT_LOG(ERR, "%p: Tx queue offloads 0x%" PRIx64
2070                         " don't match port offloads 0x%" PRIx64
2071                         " or supported port offloads 0x%" PRIx64
2072                         " or supported queue offloads 0x%" PRIx64,
2073                         (void *)dev, conf->offloads,
2074                         dev->data->dev_conf.txmode.offloads,
2075                         fm10k_get_tx_port_offloads_capa(dev),
2076                         fm10k_get_tx_queue_offloads_capa(dev));
2077                 return -ENOTSUP;
2078         }
2079
2080         /* make sure a valid number of descriptors have been requested */
2081         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
2082                                 FM10K_MULT_TX_DESC, nb_desc)) {
2083                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
2084                         "less than or equal to %"PRIu32", "
2085                         "greater than or equal to %u, "
2086                         "and a multiple of %u",
2087                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
2088                         FM10K_MULT_TX_DESC);
2089                 return -EINVAL;
2090         }
2091
2092         /*
2093          * if this queue existed already, free the associated memory. The
2094          * queue cannot be reused in case we need to allocate memory on
2095          * different socket than was previously used.
2096          */
2097         if (dev->data->tx_queues[queue_id] != NULL) {
2098                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
2099
2100                 tx_queue_free(txq);
2101                 dev->data->tx_queues[queue_id] = NULL;
2102         }
2103
2104         /* allocate memory for the queue structure */
2105         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
2106                                 socket_id);
2107         if (q == NULL) {
2108                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
2109                 return -ENOMEM;
2110         }
2111
2112         /* setup queue */
2113         q->nb_desc = nb_desc;
2114         q->port_id = dev->data->port_id;
2115         q->queue_id = queue_id;
2116         q->offloads = conf->offloads;
2117         q->ops = &def_txq_ops;
2118         q->tail_ptr = (volatile uint32_t *)
2119                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
2120         if (handle_txconf(q, conf))
2121                 return -EINVAL;
2122
2123         /* allocate memory for the software ring */
2124         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
2125                                         nb_desc * sizeof(struct rte_mbuf *),
2126                                         RTE_CACHE_LINE_SIZE, socket_id);
2127         if (q->sw_ring == NULL) {
2128                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
2129                 rte_free(q);
2130                 return -ENOMEM;
2131         }
2132
2133         /*
2134          * allocate memory for the hardware descriptor ring. A memzone large
2135          * enough to hold the maximum ring size is requested to allow for
2136          * resizing in later calls to the queue setup function.
2137          */
2138         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
2139                                       FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
2140                                       socket_id);
2141         if (mz == NULL) {
2142                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
2143                 rte_free(q->sw_ring);
2144                 rte_free(q);
2145                 return -ENOMEM;
2146         }
2147         q->hw_ring = mz->addr;
2148         q->hw_ring_phys_addr = mz->iova;
2149
2150         /*
2151          * allocate memory for the RS bit tracker. Enough slots to hold the
2152          * descriptor index for each RS bit needing to be set are required.
2153          */
2154         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
2155                                 ((nb_desc + 1) / q->rs_thresh) *
2156                                 sizeof(uint16_t),
2157                                 RTE_CACHE_LINE_SIZE, socket_id);
2158         if (q->rs_tracker.list == NULL) {
2159                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
2160                 rte_free(q->sw_ring);
2161                 rte_free(q);
2162                 return -ENOMEM;
2163         }
2164
2165         dev->data->tx_queues[queue_id] = q;
2166         return 0;
2167 }
2168
2169 static void
2170 fm10k_tx_queue_release(void *queue)
2171 {
2172         struct fm10k_tx_queue *q = queue;
2173         PMD_INIT_FUNC_TRACE();
2174
2175         tx_queue_free(q);
2176 }
2177
2178 static int
2179 fm10k_reta_update(struct rte_eth_dev *dev,
2180                         struct rte_eth_rss_reta_entry64 *reta_conf,
2181                         uint16_t reta_size)
2182 {
2183         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2184         uint16_t i, j, idx, shift;
2185         uint8_t mask;
2186         uint32_t reta;
2187
2188         PMD_INIT_FUNC_TRACE();
2189
2190         if (reta_size > FM10K_MAX_RSS_INDICES) {
2191                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2192                         "(%d) doesn't match the number hardware can supported "
2193                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2194                 return -EINVAL;
2195         }
2196
2197         /*
2198          * Update Redirection Table RETA[n], n=0..31. The redirection table has
2199          * 128-entries in 32 registers
2200          */
2201         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2202                 idx = i / RTE_RETA_GROUP_SIZE;
2203                 shift = i % RTE_RETA_GROUP_SIZE;
2204                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2205                                 BIT_MASK_PER_UINT32);
2206                 if (mask == 0)
2207                         continue;
2208
2209                 reta = 0;
2210                 if (mask != BIT_MASK_PER_UINT32)
2211                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2212
2213                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2214                         if (mask & (0x1 << j)) {
2215                                 if (mask != 0xF)
2216                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
2217                                 reta |= reta_conf[idx].reta[shift + j] <<
2218                                                 (CHAR_BIT * j);
2219                         }
2220                 }
2221                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2222         }
2223
2224         return 0;
2225 }
2226
2227 static int
2228 fm10k_reta_query(struct rte_eth_dev *dev,
2229                         struct rte_eth_rss_reta_entry64 *reta_conf,
2230                         uint16_t reta_size)
2231 {
2232         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2233         uint16_t i, j, idx, shift;
2234         uint8_t mask;
2235         uint32_t reta;
2236
2237         PMD_INIT_FUNC_TRACE();
2238
2239         if (reta_size < FM10K_MAX_RSS_INDICES) {
2240                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2241                         "(%d) doesn't match the number hardware can supported "
2242                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2243                 return -EINVAL;
2244         }
2245
2246         /*
2247          * Read Redirection Table RETA[n], n=0..31. The redirection table has
2248          * 128-entries in 32 registers
2249          */
2250         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2251                 idx = i / RTE_RETA_GROUP_SIZE;
2252                 shift = i % RTE_RETA_GROUP_SIZE;
2253                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2254                                 BIT_MASK_PER_UINT32);
2255                 if (mask == 0)
2256                         continue;
2257
2258                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2259                 for (j = 0; j < CHARS_PER_UINT32; j++) {
2260                         if (mask & (0x1 << j))
2261                                 reta_conf[idx].reta[shift + j] = ((reta >>
2262                                         CHAR_BIT * j) & UINT8_MAX);
2263                 }
2264         }
2265
2266         return 0;
2267 }
2268
2269 static int
2270 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2271         struct rte_eth_rss_conf *rss_conf)
2272 {
2273         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2274         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2275         uint32_t mrqc;
2276         uint64_t hf = rss_conf->rss_hf;
2277         int i;
2278
2279         PMD_INIT_FUNC_TRACE();
2280
2281         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2282                                 FM10K_RSSRK_ENTRIES_PER_REG))
2283                 return -EINVAL;
2284
2285         if (hf == 0)
2286                 return -EINVAL;
2287
2288         mrqc = 0;
2289         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
2290         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
2291         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
2292         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
2293         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
2294         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
2295         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
2296         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
2297         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
2298
2299         /* If the mapping doesn't fit any supported, return */
2300         if (mrqc == 0)
2301                 return -EINVAL;
2302
2303         if (key != NULL)
2304                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2305                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2306
2307         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2308
2309         return 0;
2310 }
2311
2312 static int
2313 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2314         struct rte_eth_rss_conf *rss_conf)
2315 {
2316         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2317         uint32_t *key = (uint32_t *)rss_conf->rss_key;
2318         uint32_t mrqc;
2319         uint64_t hf;
2320         int i;
2321
2322         PMD_INIT_FUNC_TRACE();
2323
2324         if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2325                                 FM10K_RSSRK_ENTRIES_PER_REG))
2326                 return -EINVAL;
2327
2328         if (key != NULL)
2329                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2330                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2331
2332         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2333         hf = 0;
2334         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
2335         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
2336         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
2337         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
2338         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
2339         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
2340         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
2341         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
2342         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
2343
2344         rss_conf->rss_hf = hf;
2345
2346         return 0;
2347 }
2348
2349 static void
2350 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2351 {
2352         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2353         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2354
2355         /* Bind all local non-queue interrupt to vector 0 */
2356         int_map |= FM10K_MISC_VEC_ID;
2357
2358         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2359         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2360         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2361         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2362         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2363         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2364
2365         /* Enable misc causes */
2366         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2367                                 FM10K_EIMR_ENABLE(THI_FAULT) |
2368                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
2369                                 FM10K_EIMR_ENABLE(MAILBOX) |
2370                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
2371                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2372                                 FM10K_EIMR_ENABLE(SRAMERROR) |
2373                                 FM10K_EIMR_ENABLE(VFLR));
2374
2375         /* Enable ITR 0 */
2376         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2377                                         FM10K_ITR_MASK_CLEAR);
2378         FM10K_WRITE_FLUSH(hw);
2379 }
2380
2381 static void
2382 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2383 {
2384         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2385         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2386
2387         int_map |= FM10K_MISC_VEC_ID;
2388
2389         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2390         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2391         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2392         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2393         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2394         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2395
2396         /* Disable misc causes */
2397         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2398                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2399                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2400                                 FM10K_EIMR_DISABLE(MAILBOX) |
2401                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2402                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2403                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2404                                 FM10K_EIMR_DISABLE(VFLR));
2405
2406         /* Disable ITR 0 */
2407         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2408         FM10K_WRITE_FLUSH(hw);
2409 }
2410
2411 static void
2412 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2413 {
2414         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2415         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2416
2417         /* Bind all local non-queue interrupt to vector 0 */
2418         int_map |= FM10K_MISC_VEC_ID;
2419
2420         /* Only INT 0 available, other 15 are reserved. */
2421         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2422
2423         /* Enable ITR 0 */
2424         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2425                                         FM10K_ITR_MASK_CLEAR);
2426         FM10K_WRITE_FLUSH(hw);
2427 }
2428
2429 static void
2430 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2431 {
2432         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2433         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2434
2435         int_map |= FM10K_MISC_VEC_ID;
2436
2437         /* Only INT 0 available, other 15 are reserved. */
2438         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2439
2440         /* Disable ITR 0 */
2441         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2442         FM10K_WRITE_FLUSH(hw);
2443 }
2444
2445 static int
2446 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2447 {
2448         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2449         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2450
2451         /* Enable ITR */
2452         if (hw->mac.type == fm10k_mac_pf)
2453                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2454                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2455         else
2456                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2457                         FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2458         rte_intr_enable(&pdev->intr_handle);
2459         return 0;
2460 }
2461
2462 static int
2463 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2464 {
2465         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2466         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2467
2468         /* Disable ITR */
2469         if (hw->mac.type == fm10k_mac_pf)
2470                 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)),
2471                         FM10K_ITR_MASK_SET);
2472         else
2473                 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)),
2474                         FM10K_ITR_MASK_SET);
2475         return 0;
2476 }
2477
2478 static int
2479 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2480 {
2481         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2482         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
2483         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
2484         uint32_t intr_vector, vec;
2485         uint16_t queue_id;
2486         int result = 0;
2487
2488         /* fm10k needs one separate interrupt for mailbox,
2489          * so only drivers which support multiple interrupt vectors
2490          * e.g. vfio-pci can work for fm10k interrupt mode
2491          */
2492         if (!rte_intr_cap_multiple(intr_handle) ||
2493                         dev->data->dev_conf.intr_conf.rxq == 0)
2494                 return result;
2495
2496         intr_vector = dev->data->nb_rx_queues;
2497
2498         /* disable interrupt first */
2499         rte_intr_disable(intr_handle);
2500         if (hw->mac.type == fm10k_mac_pf)
2501                 fm10k_dev_disable_intr_pf(dev);
2502         else
2503                 fm10k_dev_disable_intr_vf(dev);
2504
2505         if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2506                 PMD_INIT_LOG(ERR, "Failed to init event fd");
2507                 result = -EIO;
2508         }
2509
2510         if (rte_intr_dp_is_en(intr_handle) && !result) {
2511                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2512                         dev->data->nb_rx_queues * sizeof(int), 0);
2513                 if (intr_handle->intr_vec) {
2514                         for (queue_id = 0, vec = FM10K_RX_VEC_START;
2515                                         queue_id < dev->data->nb_rx_queues;
2516                                         queue_id++) {
2517                                 intr_handle->intr_vec[queue_id] = vec;
2518                                 if (vec < intr_handle->nb_efd - 1
2519                                                 + FM10K_RX_VEC_START)
2520                                         vec++;
2521                         }
2522                 } else {
2523                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2524                                 " intr_vec", dev->data->nb_rx_queues);
2525                         rte_intr_efd_disable(intr_handle);
2526                         result = -ENOMEM;
2527                 }
2528         }
2529
2530         if (hw->mac.type == fm10k_mac_pf)
2531                 fm10k_dev_enable_intr_pf(dev);
2532         else
2533                 fm10k_dev_enable_intr_vf(dev);
2534         rte_intr_enable(intr_handle);
2535         hw->mac.ops.update_int_moderator(hw);
2536         return result;
2537 }
2538
2539 static int
2540 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2541 {
2542         struct fm10k_fault fault;
2543         int err;
2544         const char *estr = "Unknown error";
2545
2546         /* Process PCA fault */
2547         if (eicr & FM10K_EICR_PCA_FAULT) {
2548                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2549                 if (err)
2550                         goto error;
2551                 switch (fault.type) {
2552                 case PCA_NO_FAULT:
2553                         estr = "PCA_NO_FAULT"; break;
2554                 case PCA_UNMAPPED_ADDR:
2555                         estr = "PCA_UNMAPPED_ADDR"; break;
2556                 case PCA_BAD_QACCESS_PF:
2557                         estr = "PCA_BAD_QACCESS_PF"; break;
2558                 case PCA_BAD_QACCESS_VF:
2559                         estr = "PCA_BAD_QACCESS_VF"; break;
2560                 case PCA_MALICIOUS_REQ:
2561                         estr = "PCA_MALICIOUS_REQ"; break;
2562                 case PCA_POISONED_TLP:
2563                         estr = "PCA_POISONED_TLP"; break;
2564                 case PCA_TLP_ABORT:
2565                         estr = "PCA_TLP_ABORT"; break;
2566                 default:
2567                         goto error;
2568                 }
2569                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2570                         estr, fault.func ? "VF" : "PF", fault.func,
2571                         fault.address, fault.specinfo);
2572         }
2573
2574         /* Process THI fault */
2575         if (eicr & FM10K_EICR_THI_FAULT) {
2576                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2577                 if (err)
2578                         goto error;
2579                 switch (fault.type) {
2580                 case THI_NO_FAULT:
2581                         estr = "THI_NO_FAULT"; break;
2582                 case THI_MAL_DIS_Q_FAULT:
2583                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2584                 default:
2585                         goto error;
2586                 }
2587                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2588                         estr, fault.func ? "VF" : "PF", fault.func,
2589                         fault.address, fault.specinfo);
2590         }
2591
2592         /* Process FUM fault */
2593         if (eicr & FM10K_EICR_FUM_FAULT) {
2594                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2595                 if (err)
2596                         goto error;
2597                 switch (fault.type) {
2598                 case FUM_NO_FAULT:
2599                         estr = "FUM_NO_FAULT"; break;
2600                 case FUM_UNMAPPED_ADDR:
2601                         estr = "FUM_UNMAPPED_ADDR"; break;
2602                 case FUM_POISONED_TLP:
2603                         estr = "FUM_POISONED_TLP"; break;
2604                 case FUM_BAD_VF_QACCESS:
2605                         estr = "FUM_BAD_VF_QACCESS"; break;
2606                 case FUM_ADD_DECODE_ERR:
2607                         estr = "FUM_ADD_DECODE_ERR"; break;
2608                 case FUM_RO_ERROR:
2609                         estr = "FUM_RO_ERROR"; break;
2610                 case FUM_QPRC_CRC_ERROR:
2611                         estr = "FUM_QPRC_CRC_ERROR"; break;
2612                 case FUM_CSR_TIMEOUT:
2613                         estr = "FUM_CSR_TIMEOUT"; break;
2614                 case FUM_INVALID_TYPE:
2615                         estr = "FUM_INVALID_TYPE"; break;
2616                 case FUM_INVALID_LENGTH:
2617                         estr = "FUM_INVALID_LENGTH"; break;
2618                 case FUM_INVALID_BE:
2619                         estr = "FUM_INVALID_BE"; break;
2620                 case FUM_INVALID_ALIGN:
2621                         estr = "FUM_INVALID_ALIGN"; break;
2622                 default:
2623                         goto error;
2624                 }
2625                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2626                         estr, fault.func ? "VF" : "PF", fault.func,
2627                         fault.address, fault.specinfo);
2628         }
2629
2630         return 0;
2631 error:
2632         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2633         return err;
2634 }
2635
2636 /**
2637  * PF interrupt handler triggered by NIC for handling specific interrupt.
2638  *
2639  * @param handle
2640  *  Pointer to interrupt handle.
2641  * @param param
2642  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2643  *
2644  * @return
2645  *  void
2646  */
2647 static void
2648 fm10k_dev_interrupt_handler_pf(void *param)
2649 {
2650         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2651         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2652         uint32_t cause, status;
2653         struct fm10k_dev_info *dev_info =
2654                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2655         int status_mbx;
2656         s32 err;
2657
2658         if (hw->mac.type != fm10k_mac_pf)
2659                 return;
2660
2661         cause = FM10K_READ_REG(hw, FM10K_EICR);
2662
2663         /* Handle PCI fault cases */
2664         if (cause & FM10K_EICR_FAULT_MASK) {
2665                 PMD_INIT_LOG(ERR, "INT: find fault!");
2666                 fm10k_dev_handle_fault(hw, cause);
2667         }
2668
2669         /* Handle switch up/down */
2670         if (cause & FM10K_EICR_SWITCHNOTREADY)
2671                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2672
2673         if (cause & FM10K_EICR_SWITCHREADY) {
2674                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2675                 if (dev_info->sm_down == 1) {
2676                         fm10k_mbx_lock(hw);
2677
2678                         /* For recreating logical ports */
2679                         status_mbx = hw->mac.ops.update_lport_state(hw,
2680                                         hw->mac.dglort_map, MAX_LPORT_NUM, 1);
2681                         if (status_mbx == FM10K_SUCCESS)
2682                                 PMD_INIT_LOG(INFO,
2683                                         "INT: Recreated Logical port");
2684                         else
2685                                 PMD_INIT_LOG(INFO,
2686                                         "INT: Logical ports weren't recreated");
2687
2688                         status_mbx = hw->mac.ops.update_xcast_mode(hw,
2689                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2690                         if (status_mbx != FM10K_SUCCESS)
2691                                 PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2692
2693                         fm10k_mbx_unlock(hw);
2694
2695                         /* first clear the internal SW recording structure */
2696                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2697                                                 ETH_MQ_RX_VMDQ_FLAG))
2698                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2699                                         false);
2700
2701                         fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2702                                         MAIN_VSI_POOL_NUMBER);
2703
2704                         /*
2705                          * Add default mac address and vlan for the logical
2706                          * ports that have been created, leave to the
2707                          * application to fully recover Rx filtering.
2708                          */
2709                         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2710                                         MAIN_VSI_POOL_NUMBER);
2711
2712                         if (!(dev->data->dev_conf.rxmode.mq_mode &
2713                                                 ETH_MQ_RX_VMDQ_FLAG))
2714                                 fm10k_vlan_filter_set(dev, hw->mac.default_vid,
2715                                         true);
2716
2717                         dev_info->sm_down = 0;
2718                         _rte_eth_dev_callback_process(dev,
2719                                         RTE_ETH_EVENT_INTR_LSC,
2720                                         NULL);
2721                 }
2722         }
2723
2724         /* Handle mailbox message */
2725         fm10k_mbx_lock(hw);
2726         err = hw->mbx.ops.process(hw, &hw->mbx);
2727         fm10k_mbx_unlock(hw);
2728
2729         if (err == FM10K_ERR_RESET_REQUESTED) {
2730                 PMD_INIT_LOG(INFO, "INT: Switch is down");
2731                 dev_info->sm_down = 1;
2732                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2733                                 NULL);
2734         }
2735
2736         /* Handle SRAM error */
2737         if (cause & FM10K_EICR_SRAMERROR) {
2738                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2739
2740                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2741                 /* Write to clear pending bits */
2742                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2743
2744                 /* Todo: print out error message after shared code  updates */
2745         }
2746
2747         /* Clear these 3 events if having any */
2748         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2749                  FM10K_EICR_SWITCHREADY;
2750         if (cause)
2751                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2752
2753         /* Re-enable interrupt from device side */
2754         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2755                                         FM10K_ITR_MASK_CLEAR);
2756         /* Re-enable interrupt from host side */
2757         rte_intr_enable(dev->intr_handle);
2758 }
2759
2760 /**
2761  * VF interrupt handler triggered by NIC for handling specific interrupt.
2762  *
2763  * @param handle
2764  *  Pointer to interrupt handle.
2765  * @param param
2766  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2767  *
2768  * @return
2769  *  void
2770  */
2771 static void
2772 fm10k_dev_interrupt_handler_vf(void *param)
2773 {
2774         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2775         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2776         struct fm10k_mbx_info *mbx = &hw->mbx;
2777         struct fm10k_dev_info *dev_info =
2778                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
2779         const enum fm10k_mbx_state state = mbx->state;
2780         int status_mbx;
2781
2782         if (hw->mac.type != fm10k_mac_vf)
2783                 return;
2784
2785         /* Handle mailbox message if lock is acquired */
2786         fm10k_mbx_lock(hw);
2787         hw->mbx.ops.process(hw, &hw->mbx);
2788         fm10k_mbx_unlock(hw);
2789
2790         if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) {
2791                 PMD_INIT_LOG(INFO, "INT: Switch has gone down");
2792
2793                 fm10k_mbx_lock(hw);
2794                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2795                                 MAX_LPORT_NUM, 1);
2796                 fm10k_mbx_unlock(hw);
2797
2798                 /* Setting reset flag */
2799                 dev_info->sm_down = 1;
2800                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2801                                 NULL);
2802         }
2803
2804         if (dev_info->sm_down == 1 &&
2805                         hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) {
2806                 PMD_INIT_LOG(INFO, "INT: Switch has gone up");
2807                 fm10k_mbx_lock(hw);
2808                 status_mbx = hw->mac.ops.update_xcast_mode(hw,
2809                                 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE);
2810                 if (status_mbx != FM10K_SUCCESS)
2811                         PMD_INIT_LOG(ERR, "Failed to set XCAST mode");
2812                 fm10k_mbx_unlock(hw);
2813
2814                 /* first clear the internal SW recording structure */
2815                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false);
2816                 fm10k_MAC_filter_set(dev, hw->mac.addr, false,
2817                                 MAIN_VSI_POOL_NUMBER);
2818
2819                 /*
2820                  * Add default mac address and vlan for the logical ports that
2821                  * have been created, leave to the application to fully recover
2822                  * Rx filtering.
2823                  */
2824                 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2825                                 MAIN_VSI_POOL_NUMBER);
2826                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
2827
2828                 dev_info->sm_down = 0;
2829                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
2830                                 NULL);
2831         }
2832
2833         /* Re-enable interrupt from device side */
2834         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2835                                         FM10K_ITR_MASK_CLEAR);
2836         /* Re-enable interrupt from host side */
2837         rte_intr_enable(dev->intr_handle);
2838 }
2839
2840 /* Mailbox message handler in VF */
2841 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2842         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2843         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2844         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2845         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2846 };
2847
2848 static int
2849 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2850 {
2851         int err = 0;
2852
2853         /* Initialize mailbox lock */
2854         fm10k_mbx_initlock(hw);
2855
2856         /* Replace default message handler with new ones */
2857         if (hw->mac.type == fm10k_mac_vf)
2858                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2859
2860         if (err) {
2861                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2862                                 err);
2863                 return err;
2864         }
2865         /* Connect to SM for PF device or PF for VF device */
2866         return hw->mbx.ops.connect(hw, &hw->mbx);
2867 }
2868
2869 static void
2870 fm10k_close_mbx_service(struct fm10k_hw *hw)
2871 {
2872         /* Disconnect from SM for PF device or PF for VF device */
2873         hw->mbx.ops.disconnect(hw, &hw->mbx);
2874 }
2875
2876 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2877         .dev_configure          = fm10k_dev_configure,
2878         .dev_start              = fm10k_dev_start,
2879         .dev_stop               = fm10k_dev_stop,
2880         .dev_close              = fm10k_dev_close,
2881         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2882         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2883         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2884         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2885         .stats_get              = fm10k_stats_get,
2886         .xstats_get             = fm10k_xstats_get,
2887         .xstats_get_names       = fm10k_xstats_get_names,
2888         .stats_reset            = fm10k_stats_reset,
2889         .xstats_reset           = fm10k_stats_reset,
2890         .link_update            = fm10k_link_update,
2891         .dev_infos_get          = fm10k_dev_infos_get,
2892         .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get,
2893         .vlan_filter_set        = fm10k_vlan_filter_set,
2894         .vlan_offload_set       = fm10k_vlan_offload_set,
2895         .mac_addr_add           = fm10k_macaddr_add,
2896         .mac_addr_remove        = fm10k_macaddr_remove,
2897         .rx_queue_start         = fm10k_dev_rx_queue_start,
2898         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2899         .tx_queue_start         = fm10k_dev_tx_queue_start,
2900         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2901         .rx_queue_setup         = fm10k_rx_queue_setup,
2902         .rx_queue_release       = fm10k_rx_queue_release,
2903         .tx_queue_setup         = fm10k_tx_queue_setup,
2904         .tx_queue_release       = fm10k_tx_queue_release,
2905         .rx_descriptor_done     = fm10k_dev_rx_descriptor_done,
2906         .rx_queue_intr_enable   = fm10k_dev_rx_queue_intr_enable,
2907         .rx_queue_intr_disable  = fm10k_dev_rx_queue_intr_disable,
2908         .reta_update            = fm10k_reta_update,
2909         .reta_query             = fm10k_reta_query,
2910         .rss_hash_update        = fm10k_rss_hash_update,
2911         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2912 };
2913
2914 static int ftag_check_handler(__rte_unused const char *key,
2915                 const char *value, __rte_unused void *opaque)
2916 {
2917         if (strcmp(value, "1"))
2918                 return -1;
2919
2920         return 0;
2921 }
2922
2923 static int
2924 fm10k_check_ftag(struct rte_devargs *devargs)
2925 {
2926         struct rte_kvargs *kvlist;
2927         const char *ftag_key = "enable_ftag";
2928
2929         if (devargs == NULL)
2930                 return 0;
2931
2932         kvlist = rte_kvargs_parse(devargs->args, NULL);
2933         if (kvlist == NULL)
2934                 return 0;
2935
2936         if (!rte_kvargs_count(kvlist, ftag_key)) {
2937                 rte_kvargs_free(kvlist);
2938                 return 0;
2939         }
2940         /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2941         if (rte_kvargs_process(kvlist, ftag_key,
2942                                 ftag_check_handler, NULL) < 0) {
2943                 rte_kvargs_free(kvlist);
2944                 return 0;
2945         }
2946         rte_kvargs_free(kvlist);
2947
2948         return 1;
2949 }
2950
2951 static uint16_t
2952 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
2953                     uint16_t nb_pkts)
2954 {
2955         uint16_t nb_tx = 0;
2956         struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
2957
2958         while (nb_pkts) {
2959                 uint16_t ret, num;
2960
2961                 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh);
2962                 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx],
2963                                                  num);
2964                 nb_tx += ret;
2965                 nb_pkts -= ret;
2966                 if (ret < num)
2967                         break;
2968         }
2969
2970         return nb_tx;
2971 }
2972
2973 static void __attribute__((cold))
2974 fm10k_set_tx_function(struct rte_eth_dev *dev)
2975 {
2976         struct fm10k_tx_queue *txq;
2977         int i;
2978         int use_sse = 1;
2979         uint16_t tx_ftag_en = 0;
2980
2981         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2982                 /* primary process has set the ftag flag and offloads */
2983                 txq = dev->data->tx_queues[0];
2984                 if (fm10k_tx_vec_condition_check(txq)) {
2985                         dev->tx_pkt_burst = fm10k_xmit_pkts;
2986                         dev->tx_pkt_prepare = fm10k_prep_pkts;
2987                         PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2988                 } else {
2989                         PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2990                         dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2991                         dev->tx_pkt_prepare = NULL;
2992                 }
2993                 return;
2994         }
2995
2996         if (fm10k_check_ftag(dev->device->devargs))
2997                 tx_ftag_en = 1;
2998
2999         for (i = 0; i < dev->data->nb_tx_queues; i++) {
3000                 txq = dev->data->tx_queues[i];
3001                 txq->tx_ftag_en = tx_ftag_en;
3002                 /* Check if Vector Tx is satisfied */
3003                 if (fm10k_tx_vec_condition_check(txq))
3004                         use_sse = 0;
3005         }
3006
3007         if (use_sse) {
3008                 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
3009                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
3010                         txq = dev->data->tx_queues[i];
3011                         fm10k_txq_vec_setup(txq);
3012                 }
3013                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
3014                 dev->tx_pkt_prepare = NULL;
3015         } else {
3016                 dev->tx_pkt_burst = fm10k_xmit_pkts;
3017                 dev->tx_pkt_prepare = fm10k_prep_pkts;
3018                 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
3019         }
3020 }
3021
3022 static void __attribute__((cold))
3023 fm10k_set_rx_function(struct rte_eth_dev *dev)
3024 {
3025         struct fm10k_dev_info *dev_info =
3026                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3027         uint16_t i, rx_using_sse;
3028         uint16_t rx_ftag_en = 0;
3029
3030         if (fm10k_check_ftag(dev->device->devargs))
3031                 rx_ftag_en = 1;
3032
3033         /* In order to allow Vector Rx there are a few configuration
3034          * conditions to be met.
3035          */
3036         if (!fm10k_rx_vec_condition_check(dev) &&
3037                         dev_info->rx_vec_allowed && !rx_ftag_en) {
3038                 if (dev->data->scattered_rx)
3039                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
3040                 else
3041                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
3042         } else if (dev->data->scattered_rx)
3043                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
3044         else
3045                 dev->rx_pkt_burst = fm10k_recv_pkts;
3046
3047         rx_using_sse =
3048                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
3049                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
3050
3051         if (rx_using_sse)
3052                 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
3053         else
3054                 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
3055
3056         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3057                 return;
3058
3059         for (i = 0; i < dev->data->nb_rx_queues; i++) {
3060                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
3061
3062                 rxq->rx_using_sse = rx_using_sse;
3063                 rxq->rx_ftag_en = rx_ftag_en;
3064         }
3065 }
3066
3067 static void
3068 fm10k_params_init(struct rte_eth_dev *dev)
3069 {
3070         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3071         struct fm10k_dev_info *info =
3072                 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private);
3073
3074         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
3075          * there is no way to get link status without reading BAR4.  Until this
3076          * works, assume we have maximum bandwidth.
3077          * @todo - fix bus info
3078          */
3079         hw->bus_caps.speed = fm10k_bus_speed_8000;
3080         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
3081         hw->bus_caps.payload = fm10k_bus_payload_512;
3082         hw->bus.speed = fm10k_bus_speed_8000;
3083         hw->bus.width = fm10k_bus_width_pcie_x8;
3084         hw->bus.payload = fm10k_bus_payload_256;
3085
3086         info->rx_vec_allowed = true;
3087 }
3088
3089 static int
3090 eth_fm10k_dev_init(struct rte_eth_dev *dev)
3091 {
3092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3093         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3094         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3095         int diag, i;
3096         struct fm10k_macvlan_filter_info *macvlan;
3097
3098         PMD_INIT_FUNC_TRACE();
3099
3100         dev->dev_ops = &fm10k_eth_dev_ops;
3101         dev->rx_pkt_burst = &fm10k_recv_pkts;
3102         dev->tx_pkt_burst = &fm10k_xmit_pkts;
3103         dev->tx_pkt_prepare = &fm10k_prep_pkts;
3104
3105         /*
3106          * Primary process does the whole initialization, for secondary
3107          * processes, we just select the same Rx and Tx function as primary.
3108          */
3109         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3110                 fm10k_set_rx_function(dev);
3111                 fm10k_set_tx_function(dev);
3112                 return 0;
3113         }
3114
3115         rte_eth_copy_pci_info(dev, pdev);
3116
3117         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
3118         memset(macvlan, 0, sizeof(*macvlan));
3119         /* Vendor and Device ID need to be set before init of shared code */
3120         memset(hw, 0, sizeof(*hw));
3121         hw->device_id = pdev->id.device_id;
3122         hw->vendor_id = pdev->id.vendor_id;
3123         hw->subsystem_device_id = pdev->id.subsystem_device_id;
3124         hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id;
3125         hw->revision_id = 0;
3126         hw->hw_addr = (void *)pdev->mem_resource[0].addr;
3127         if (hw->hw_addr == NULL) {
3128                 PMD_INIT_LOG(ERR, "Bad mem resource."
3129                         " Try to blacklist unused devices.");
3130                 return -EIO;
3131         }
3132
3133         /* Store fm10k_adapter pointer */
3134         hw->back = dev->data->dev_private;
3135
3136         /* Initialize the shared code */
3137         diag = fm10k_init_shared_code(hw);
3138         if (diag != FM10K_SUCCESS) {
3139                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
3140                 return -EIO;
3141         }
3142
3143         /* Initialize parameters */
3144         fm10k_params_init(dev);
3145
3146         /* Initialize the hw */
3147         diag = fm10k_init_hw(hw);
3148         if (diag != FM10K_SUCCESS) {
3149                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
3150                 return -EIO;
3151         }
3152
3153         /* Initialize MAC address(es) */
3154         dev->data->mac_addrs = rte_zmalloc("fm10k",
3155                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
3156         if (dev->data->mac_addrs == NULL) {
3157                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
3158                 return -ENOMEM;
3159         }
3160
3161         diag = fm10k_read_mac_addr(hw);
3162
3163         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3164                         &dev->data->mac_addrs[0]);
3165
3166         if (diag != FM10K_SUCCESS ||
3167                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
3168
3169                 /* Generate a random addr */
3170                 eth_random_addr(hw->mac.addr);
3171                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
3172                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
3173                 &dev->data->mac_addrs[0]);
3174         }
3175
3176         /* Reset the hw statistics */
3177         fm10k_stats_reset(dev);
3178
3179         /* Reset the hw */
3180         diag = fm10k_reset_hw(hw);
3181         if (diag != FM10K_SUCCESS) {
3182                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
3183                 return -EIO;
3184         }
3185
3186         /* Setup mailbox service */
3187         diag = fm10k_setup_mbx_service(hw);
3188         if (diag != FM10K_SUCCESS) {
3189                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
3190                 return -EIO;
3191         }
3192
3193         /*PF/VF has different interrupt handling mechanism */
3194         if (hw->mac.type == fm10k_mac_pf) {
3195                 /* register callback func to eal lib */
3196                 rte_intr_callback_register(intr_handle,
3197                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3198
3199                 /* enable MISC interrupt */
3200                 fm10k_dev_enable_intr_pf(dev);
3201         } else { /* VF */
3202                 rte_intr_callback_register(intr_handle,
3203                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3204
3205                 fm10k_dev_enable_intr_vf(dev);
3206         }
3207
3208         /* Enable intr after callback registered */
3209         rte_intr_enable(intr_handle);
3210
3211         hw->mac.ops.update_int_moderator(hw);
3212
3213         /* Make sure Switch Manager is ready before going forward. */
3214         if (hw->mac.type == fm10k_mac_pf) {
3215                 int switch_ready = 0;
3216
3217                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3218                         fm10k_mbx_lock(hw);
3219                         hw->mac.ops.get_host_state(hw, &switch_ready);
3220                         fm10k_mbx_unlock(hw);
3221                         if (switch_ready)
3222                                 break;
3223                         /* Delay some time to acquire async LPORT_MAP info. */
3224                         rte_delay_us(WAIT_SWITCH_MSG_US);
3225                 }
3226
3227                 if (switch_ready == 0) {
3228                         PMD_INIT_LOG(ERR, "switch is not ready");
3229                         return -1;
3230                 }
3231         }
3232
3233         /*
3234          * Below function will trigger operations on mailbox, acquire lock to
3235          * avoid race condition from interrupt handler. Operations on mailbox
3236          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
3237          * will handle and generate an interrupt to our side. Then,  FIFO in
3238          * mailbox will be touched.
3239          */
3240         fm10k_mbx_lock(hw);
3241         /* Enable port first */
3242         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
3243                                         MAX_LPORT_NUM, 1);
3244
3245         /* Set unicast mode by default. App can change to other mode in other
3246          * API func.
3247          */
3248         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
3249                                         FM10K_XCAST_MODE_NONE);
3250
3251         fm10k_mbx_unlock(hw);
3252
3253         /* Make sure default VID is ready before going forward. */
3254         if (hw->mac.type == fm10k_mac_pf) {
3255                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
3256                         if (hw->mac.default_vid)
3257                                 break;
3258                         /* Delay some time to acquire async port VLAN info. */
3259                         rte_delay_us(WAIT_SWITCH_MSG_US);
3260                 }
3261
3262                 if (!hw->mac.default_vid) {
3263                         PMD_INIT_LOG(ERR, "default VID is not ready");
3264                         return -1;
3265                 }
3266         }
3267
3268         /* Add default mac address */
3269         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
3270                 MAIN_VSI_POOL_NUMBER);
3271
3272         return 0;
3273 }
3274
3275 static int
3276 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
3277 {
3278         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3279         struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev);
3280         struct rte_intr_handle *intr_handle = &pdev->intr_handle;
3281         PMD_INIT_FUNC_TRACE();
3282
3283         /* only uninitialize in the primary process */
3284         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3285                 return 0;
3286
3287         /* safe to close dev here */
3288         fm10k_dev_close(dev);
3289
3290         dev->dev_ops = NULL;
3291         dev->rx_pkt_burst = NULL;
3292         dev->tx_pkt_burst = NULL;
3293
3294         /* disable uio/vfio intr */
3295         rte_intr_disable(intr_handle);
3296
3297         /*PF/VF has different interrupt handling mechanism */
3298         if (hw->mac.type == fm10k_mac_pf) {
3299                 /* disable interrupt */
3300                 fm10k_dev_disable_intr_pf(dev);
3301
3302                 /* unregister callback func to eal lib */
3303                 rte_intr_callback_unregister(intr_handle,
3304                         fm10k_dev_interrupt_handler_pf, (void *)dev);
3305         } else {
3306                 /* disable interrupt */
3307                 fm10k_dev_disable_intr_vf(dev);
3308
3309                 rte_intr_callback_unregister(intr_handle,
3310                         fm10k_dev_interrupt_handler_vf, (void *)dev);
3311         }
3312
3313         /* free mac memory */
3314         if (dev->data->mac_addrs) {
3315                 rte_free(dev->data->mac_addrs);
3316                 dev->data->mac_addrs = NULL;
3317         }
3318
3319         memset(hw, 0, sizeof(*hw));
3320
3321         return 0;
3322 }
3323
3324 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3325         struct rte_pci_device *pci_dev)
3326 {
3327         return rte_eth_dev_pci_generic_probe(pci_dev,
3328                 sizeof(struct fm10k_adapter), eth_fm10k_dev_init);
3329 }
3330
3331 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev)
3332 {
3333         return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit);
3334 }
3335
3336 /*
3337  * The set of PCI devices this driver supports. This driver will enable both PF
3338  * and SRIOV-VF devices.
3339  */
3340 static const struct rte_pci_id pci_id_fm10k_map[] = {
3341         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) },
3342         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) },
3343         { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) },
3344         { .vendor_id = 0, /* sentinel */ },
3345 };
3346
3347 static struct rte_pci_driver rte_pmd_fm10k = {
3348         .id_table = pci_id_fm10k_map,
3349         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3350                      RTE_PCI_DRV_IOVA_AS_VA,
3351         .probe = eth_fm10k_pci_probe,
3352         .remove = eth_fm10k_pci_remove,
3353 };
3354
3355 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k);
3356 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map);
3357 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci");
3358
3359 RTE_INIT(fm10k_init_log);
3360 static void
3361 fm10k_init_log(void)
3362 {
3363         fm10k_logtype_init = rte_log_register("pmd.net.fm10k.init");
3364         if (fm10k_logtype_init >= 0)
3365                 rte_log_set_level(fm10k_logtype_init, RTE_LOG_NOTICE);
3366         fm10k_logtype_driver = rte_log_register("pmd.net.fm10k.driver");
3367         if (fm10k_logtype_driver >= 0)
3368                 rte_log_set_level(fm10k_logtype_driver, RTE_LOG_NOTICE);
3369 }