drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 #define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
  59                                 ETH_TXQ_FLAGS_NOOFFLOADS)
  60
  61 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  62 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  63 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  64 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  65 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  66 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  67 static int
  68 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  69 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  70         const u8 *mac, bool add, uint32_t pool);
  71 static void fm10k_tx_queue_release(void *queue);
  72 static void fm10k_rx_queue_release(void *queue);
  73 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  74 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  75
  76 static void
  77 fm10k_mbx_initlock(struct fm10k_hw *hw)
  78 {
  79         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  80 }
  81
  82 static void
  83 fm10k_mbx_lock(struct fm10k_hw *hw)
  84 {
  85         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  86                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  87 }
  88
  89 static void
  90 fm10k_mbx_unlock(struct fm10k_hw *hw)
  91 {
  92         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  93 }
  94
  95 /*
  96  * reset queue to initial state, allocate software buffers used when starting
  97  * device.
  98  * return 0 on success
  99  * return -ENOMEM if buffers cannot be allocated
 100  * return -EINVAL if buffers do not satisfy alignment condition
 101  */
 102 static inline int
 103 rx_queue_reset(struct fm10k_rx_queue *q)
 104 {
 105         static const union fm10k_rx_desc zero = {{0} };
 106         uint64_t dma_addr;
 107         int i, diag;
 108         PMD_INIT_FUNC_TRACE();
 109
 110         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 111         if (diag != 0)
 112                 return -ENOMEM;
 113
 114         for (i = 0; i < q->nb_desc; ++i) {
 115                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 116                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 117                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 118                                                 q->nb_desc);
 119                         return -EINVAL;
 120                 }
 121                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 122                 q->hw_ring[i].q.pkt_addr = dma_addr;
 123                 q->hw_ring[i].q.hdr_addr = dma_addr;
 124         }
 125
 126         /* initialize extra software ring entries. Space for these extra
 127          * entries is always allocated.
 128          */
 129         memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
 130         for (i = 0; i < q->nb_fake_desc; ++i) {
 131                 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
 132                 q->hw_ring[q->nb_desc + i] = zero;
 133         }
 134
 135         q->next_dd = 0;
 136         q->next_alloc = 0;
 137         q->next_trigger = q->alloc_thresh - 1;
 138         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 139         q->rxrearm_start = 0;
 140         q->rxrearm_nb = 0;
 141
 142         return 0;
 143 }
 144
 145 /*
 146  * clean queue, descriptor rings, free software buffers used when stopping
 147  * device.
 148  */
 149 static inline void
 150 rx_queue_clean(struct fm10k_rx_queue *q)
 151 {
 152         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 153         uint32_t i;
 154         PMD_INIT_FUNC_TRACE();
 155
 156         /* zero descriptor rings */
 157         for (i = 0; i < q->nb_desc; ++i)
 158                 q->hw_ring[i] = zero;
 159
 160         /* zero faked descriptors */
 161         for (i = 0; i < q->nb_fake_desc; ++i)
 162                 q->hw_ring[q->nb_desc + i] = zero;
 163
 164         /* vPMD driver has a different way of releasing mbufs. */
 165         if (q->rx_using_sse) {
 166                 fm10k_rx_queue_release_mbufs_vec(q);
 167                 return;
 168         }
 169
 170         /* free software buffers */
 171         for (i = 0; i < q->nb_desc; ++i) {
 172                 if (q->sw_ring[i]) {
 173                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 174                         q->sw_ring[i] = NULL;
 175                 }
 176         }
 177 }
 178
 179 /*
 180  * free all queue memory used when releasing the queue (i.e. configure)
 181  */
 182 static inline void
 183 rx_queue_free(struct fm10k_rx_queue *q)
 184 {
 185         PMD_INIT_FUNC_TRACE();
 186         if (q) {
 187                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 188                 rx_queue_clean(q);
 189                 if (q->sw_ring) {
 190                         rte_free(q->sw_ring);
 191                         q->sw_ring = NULL;
 192                 }
 193                 rte_free(q);
 194                 q = NULL;
 195         }
 196 }
 197
 198 /*
 199  * disable RX queue, wait unitl HW finished necessary flush operation
 200  */
 201 static inline int
 202 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 203 {
 204         uint32_t reg, i;
 205
 206         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 207         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 208                         reg & ~FM10K_RXQCTL_ENABLE);
 209
 210         /* Wait 100us at most */
 211         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 212                 rte_delay_us(1);
 213                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 214                 if (!(reg & FM10K_RXQCTL_ENABLE))
 215                         break;
 216         }
 217
 218         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 219                 return -1;
 220
 221         return 0;
 222 }
 223
 224 /*
 225  * reset queue to initial state, allocate software buffers used when starting
 226  * device
 227  */
 228 static inline void
 229 tx_queue_reset(struct fm10k_tx_queue *q)
 230 {
 231         PMD_INIT_FUNC_TRACE();
 232         q->last_free = 0;
 233         q->next_free = 0;
 234         q->nb_used = 0;
 235         q->nb_free = q->nb_desc - 1;
 236         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 237         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 238 }
 239
 240 /*
 241  * clean queue, descriptor rings, free software buffers used when stopping
 242  * device
 243  */
 244 static inline void
 245 tx_queue_clean(struct fm10k_tx_queue *q)
 246 {
 247         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 248         uint32_t i;
 249         PMD_INIT_FUNC_TRACE();
 250
 251         /* zero descriptor rings */
 252         for (i = 0; i < q->nb_desc; ++i)
 253                 q->hw_ring[i] = zero;
 254
 255         /* free software buffers */
 256         for (i = 0; i < q->nb_desc; ++i) {
 257                 if (q->sw_ring[i]) {
 258                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 259                         q->sw_ring[i] = NULL;
 260                 }
 261         }
 262 }
 263
 264 /*
 265  * free all queue memory used when releasing the queue (i.e. configure)
 266  */
 267 static inline void
 268 tx_queue_free(struct fm10k_tx_queue *q)
 269 {
 270         PMD_INIT_FUNC_TRACE();
 271         if (q) {
 272                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 273                 tx_queue_clean(q);
 274                 if (q->rs_tracker.list) {
 275                         rte_free(q->rs_tracker.list);
 276                         q->rs_tracker.list = NULL;
 277                 }
 278                 if (q->sw_ring) {
 279                         rte_free(q->sw_ring);
 280                         q->sw_ring = NULL;
 281                 }
 282                 rte_free(q);
 283                 q = NULL;
 284         }
 285 }
 286
 287 /*
 288  * disable TX queue, wait unitl HW finished necessary flush operation
 289  */
 290 static inline int
 291 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 292 {
 293         uint32_t reg, i;
 294
 295         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 296         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 297                         reg & ~FM10K_TXDCTL_ENABLE);
 298
 299         /* Wait 100us at most */
 300         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 301                 rte_delay_us(1);
 302                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 303                 if (!(reg & FM10K_TXDCTL_ENABLE))
 304                         break;
 305         }
 306
 307         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 308                 return -1;
 309
 310         return 0;
 311 }
 312
 313 static int
 314 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 315 {
 316         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 317         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 318         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 319         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 320
 321         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 322
 323         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 324                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 325                 return -EINVAL;
 326         }
 327
 328         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 329                 return 0;
 330
 331         if (hw->mac.type == fm10k_mac_vf) {
 332                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 333                 return -EINVAL;
 334         }
 335
 336         /* Check VMDQ queue pool number */
 337         if (vmdq_conf->nb_queue_pools >
 338                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 339                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 340                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 341                         vmdq_conf->nb_queue_pools);
 342                 return -EINVAL;
 343         }
 344
 345         return 0;
 346 }
 347
 348 static const struct fm10k_txq_ops def_txq_ops = {
 349         .release_mbufs = tx_queue_free,
 350         .reset = tx_queue_reset,
 351 };
 352
 353 static int
 354 fm10k_dev_configure(struct rte_eth_dev *dev)
 355 {
 356         int ret;
 357
 358         PMD_INIT_FUNC_TRACE();
 359
 360         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 361                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 362         /* multipe queue mode checking */
 363         ret  = fm10k_check_mq_mode(dev);
 364         if (ret != 0) {
 365                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 366                             ret);
 367                 return ret;
 368         }
 369
 370         return 0;
 371 }
 372
 373 /* fls = find last set bit = 32 minus the number of leading zeros */
 374 #ifndef fls
 375 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 376 #endif
 377
 378 static void
 379 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 380 {
 381         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 382         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 383         uint32_t i;
 384
 385         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 386
 387         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 388                 if (!vmdq_conf->pool_map[i].pools)
 389                         continue;
 390                 fm10k_mbx_lock(hw);
 391                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 392                 fm10k_mbx_unlock(hw);
 393         }
 394 }
 395
 396 static void
 397 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 398 {
 399         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 400
 401         /* Add default mac address */
 402         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 403                 MAIN_VSI_POOL_NUMBER);
 404 }
 405
 406 static void
 407 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 408 {
 409         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 410         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 411         uint32_t mrqc, *key, i, reta, j;
 412         uint64_t hf;
 413
 414 #define RSS_KEY_SIZE 40
 415         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 416                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 417                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 418                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 419                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 420                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 421         };
 422
 423         if (dev->data->nb_rx_queues == 1 ||
 424             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 425             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 426                 return;
 427
 428         /* random key is rss_intel_key (default) or user provided (rss_key) */
 429         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 430                 key = (uint32_t *)rss_intel_key;
 431         else
 432                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 433
 434         /* Now fill our hash function seeds, 4 bytes at a time */
 435         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 436                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 437
 438         /*
 439          * Fill in redirection table
 440          * The byte-swap is needed because NIC registers are in
 441          * little-endian order.
 442          */
 443         reta = 0;
 444         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 445                 if (j == dev->data->nb_rx_queues)
 446                         j = 0;
 447                 reta = (reta << CHAR_BIT) | j;
 448                 if ((i & 3) == 3)
 449                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 450                                         rte_bswap32(reta));
 451         }
 452
 453         /*
 454          * Generate RSS hash based on packet types, TCP/UDP
 455          * port numbers and/or IPv4/v6 src and dst addresses
 456          */
 457         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 458         mrqc = 0;
 459         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 460         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 461         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 462         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 463         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 464         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 465         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 466         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 467         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 468
 469         if (mrqc == 0) {
 470                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 471                         "supported", hf);
 472                 return;
 473         }
 474
 475         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 476 }
 477
 478 static void
 479 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 480         uint16_t nb_lport_old, uint16_t nb_lport_new)
 481 {
 482         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 483         uint32_t i;
 484
 485         fm10k_mbx_lock(hw);
 486         /* Disable previous logic ports */
 487         if (nb_lport_old)
 488                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 489                         nb_lport_old, false);
 490         /* Enable new logic ports */
 491         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 492                 nb_lport_new, true);
 493         fm10k_mbx_unlock(hw);
 494
 495         for (i = 0; i < nb_lport_new; i++) {
 496                 /* Set unicast mode by default. App can change
 497                  * to other mode in other API func.
 498                  */
 499                 fm10k_mbx_lock(hw);
 500                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 501                         FM10K_XCAST_MODE_NONE);
 502                 fm10k_mbx_unlock(hw);
 503         }
 504 }
 505
 506 static void
 507 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 508 {
 509         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 510         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 511         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 512         struct fm10k_macvlan_filter_info *macvlan;
 513         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 514         uint16_t nb_lport_new, nb_lport_old;
 515
 516         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 517         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 518
 519         fm10k_dev_rss_configure(dev);
 520
 521         /* only PF supports VMDQ */
 522         if (hw->mac.type != fm10k_mac_pf)
 523                 return;
 524
 525         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 526                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 527
 528         /* no pool number change, no need to update logic port and VLAN/MAC */
 529         if (macvlan->nb_queue_pools == nb_queue_pools)
 530                 return;
 531
 532         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 533         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 534         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 535
 536         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 537         memset(dev->data->mac_addrs, 0,
 538                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 539         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 540                 &dev->data->mac_addrs[0]);
 541         memset(macvlan, 0, sizeof(*macvlan));
 542         macvlan->nb_queue_pools = nb_queue_pools;
 543
 544         if (nb_queue_pools)
 545                 fm10k_dev_vmdq_rx_configure(dev);
 546         else
 547                 fm10k_dev_pf_main_vsi_reset(dev);
 548 }
 549
 550 static int
 551 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 552 {
 553         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 554         int i, ret;
 555         struct fm10k_tx_queue *txq;
 556         uint64_t base_addr;
 557         uint32_t size;
 558
 559         /* Disable TXINT to avoid possible interrupt */
 560         for (i = 0; i < hw->mac.max_queues; i++)
 561                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 562                                 3 << FM10K_TXINT_TIMER_SHIFT);
 563
 564         /* Setup TX queue */
 565         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 566                 txq = dev->data->tx_queues[i];
 567                 base_addr = txq->hw_ring_phys_addr;
 568                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 569
 570                 /* disable queue to avoid issues while updating state */
 571                 ret = tx_queue_disable(hw, i);
 572                 if (ret) {
 573                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 574                         return -1;
 575                 }
 576
 577                 /* set location and size for descriptor ring */
 578                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 579                                 base_addr & UINT64_LOWER_32BITS_MASK);
 580                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 581                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 582                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 583         }
 584
 585         /* set up vector or scalar TX function as appropriate */
 586         fm10k_set_tx_function(dev);
 587
 588         return 0;
 589 }
 590
 591 static int
 592 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 593 {
 594         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 595         int i, ret;
 596         struct fm10k_rx_queue *rxq;
 597         uint64_t base_addr;
 598         uint32_t size;
 599         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 600         uint16_t buf_size;
 601
 602         /* Disable RXINT to avoid possible interrupt */
 603         for (i = 0; i < hw->mac.max_queues; i++)
 604                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 605                                 3 << FM10K_RXINT_TIMER_SHIFT);
 606
 607         /* Setup RX queues */
 608         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 609                 rxq = dev->data->rx_queues[i];
 610                 base_addr = rxq->hw_ring_phys_addr;
 611                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 612
 613                 /* disable queue to avoid issues while updating state */
 614                 ret = rx_queue_disable(hw, i);
 615                 if (ret) {
 616                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 617                         return -1;
 618                 }
 619
 620                 /* Setup the Base and Length of the Rx Descriptor Ring */
 621                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 622                                 base_addr & UINT64_LOWER_32BITS_MASK);
 623                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 624                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 625                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 626
 627                 /* Configure the Rx buffer size for one buff without split */
 628                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 629                         RTE_PKTMBUF_HEADROOM);
 630                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 631                  * reserved for this purpose, and the worst case could be 511B.
 632                  * But SRR reg assumes all buffers have the same size. In order
 633                  * to fill the gap, we'll have to consider the worst case and
 634                  * assume 512B is reserved. If we don't do so, it's possible
 635                  * for HW to overwrite data to next mbuf.
 636                  */
 637                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 638
 639                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 640                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 641
 642                 /* It adds dual VLAN length for supporting dual VLAN */
 643                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 644                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 645                         dev->data->dev_conf.rxmode.enable_scatter) {
 646                         uint32_t reg;
 647                         dev->data->scattered_rx = 1;
 648                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 649                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 650                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 651                 }
 652
 653                 /* Enable drop on empty, it's RO for VF */
 654                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 655                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 656
 657                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 658                 FM10K_WRITE_FLUSH(hw);
 659         }
 660
 661         /* Configure VMDQ/RSS if applicable */
 662         fm10k_dev_mq_rx_configure(dev);
 663
 664         /* Decide the best RX function */
 665         fm10k_set_rx_function(dev);
 666
 667         return 0;
 668 }
 669
 670 static int
 671 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 672 {
 673         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 674         int err = -1;
 675         uint32_t reg;
 676         struct fm10k_rx_queue *rxq;
 677
 678         PMD_INIT_FUNC_TRACE();
 679
 680         if (rx_queue_id < dev->data->nb_rx_queues) {
 681                 rxq = dev->data->rx_queues[rx_queue_id];
 682                 err = rx_queue_reset(rxq);
 683                 if (err == -ENOMEM) {
 684                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 685                         return err;
 686                 } else if (err == -EINVAL) {
 687                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 688                                 " %d", err);
 689                         return err;
 690                 }
 691
 692                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 693                  * Note: this must be done AFTER the queue is enabled on real
 694                  * hardware, but BEFORE the queue is enabled when using the
 695                  * emulation platform. Do it in both places for now and remove
 696                  * this comment and the following two register writes when the
 697                  * emulation platform is no longer being used.
 698                  */
 699                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 700                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 701
 702                 /* Set PF ownership flag for PF devices */
 703                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 704                 if (hw->mac.type == fm10k_mac_pf)
 705                         reg |= FM10K_RXQCTL_PF;
 706                 reg |= FM10K_RXQCTL_ENABLE;
 707                 /* enable RX queue */
 708                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 709                 FM10K_WRITE_FLUSH(hw);
 710
 711                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 712                  * Note: this must be done AFTER the queue is enabled
 713                  */
 714                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 715                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 716         }
 717
 718         return err;
 719 }
 720
 721 static int
 722 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 723 {
 724         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 725
 726         PMD_INIT_FUNC_TRACE();
 727
 728         if (rx_queue_id < dev->data->nb_rx_queues) {
 729                 /* Disable RX queue */
 730                 rx_queue_disable(hw, rx_queue_id);
 731
 732                 /* Free mbuf and clean HW ring */
 733                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 734         }
 735
 736         return 0;
 737 }
 738
 739 static int
 740 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 741 {
 742         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 743         /** @todo - this should be defined in the shared code */
 744 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 745         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 746         int err = 0;
 747
 748         PMD_INIT_FUNC_TRACE();
 749
 750         if (tx_queue_id < dev->data->nb_tx_queues) {
 751                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 752
 753                 q->ops->reset(q);
 754
 755                 /* reset head and tail pointers */
 756                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 757                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 758
 759                 /* enable TX queue */
 760                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 761                                         FM10K_TXDCTL_ENABLE | txdctl);
 762                 FM10K_WRITE_FLUSH(hw);
 763         } else
 764                 err = -1;
 765
 766         return err;
 767 }
 768
 769 static int
 770 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 771 {
 772         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 773
 774         PMD_INIT_FUNC_TRACE();
 775
 776         if (tx_queue_id < dev->data->nb_tx_queues) {
 777                 tx_queue_disable(hw, tx_queue_id);
 778                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 779         }
 780
 781         return 0;
 782 }
 783
 784 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 785 {
 786         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 787                 != FM10K_DGLORTMAP_NONE);
 788 }
 789
 790 static void
 791 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 792 {
 793         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 794         int status;
 795
 796         PMD_INIT_FUNC_TRACE();
 797
 798         /* Return if it didn't acquire valid glort range */
 799         if (!fm10k_glort_valid(hw))
 800                 return;
 801
 802         fm10k_mbx_lock(hw);
 803         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 804                                 FM10K_XCAST_MODE_PROMISC);
 805         fm10k_mbx_unlock(hw);
 806
 807         if (status != FM10K_SUCCESS)
 808                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 809 }
 810
 811 static void
 812 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 813 {
 814         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 815         uint8_t mode;
 816         int status;
 817
 818         PMD_INIT_FUNC_TRACE();
 819
 820         /* Return if it didn't acquire valid glort range */
 821         if (!fm10k_glort_valid(hw))
 822                 return;
 823
 824         if (dev->data->all_multicast == 1)
 825                 mode = FM10K_XCAST_MODE_ALLMULTI;
 826         else
 827                 mode = FM10K_XCAST_MODE_NONE;
 828
 829         fm10k_mbx_lock(hw);
 830         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 831                                 mode);
 832         fm10k_mbx_unlock(hw);
 833
 834         if (status != FM10K_SUCCESS)
 835                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 836 }
 837
 838 static void
 839 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 840 {
 841         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 842         int status;
 843
 844         PMD_INIT_FUNC_TRACE();
 845
 846         /* Return if it didn't acquire valid glort range */
 847         if (!fm10k_glort_valid(hw))
 848                 return;
 849
 850         /* If promiscuous mode is enabled, it doesn't make sense to enable
 851          * allmulticast and disable promiscuous since fm10k only can select
 852          * one of the modes.
 853          */
 854         if (dev->data->promiscuous) {
 855                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 856                         "needn't enable allmulticast");
 857                 return;
 858         }
 859
 860         fm10k_mbx_lock(hw);
 861         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 862                                 FM10K_XCAST_MODE_ALLMULTI);
 863         fm10k_mbx_unlock(hw);
 864
 865         if (status != FM10K_SUCCESS)
 866                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 867 }
 868
 869 static void
 870 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 871 {
 872         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 873         int status;
 874
 875         PMD_INIT_FUNC_TRACE();
 876
 877         /* Return if it didn't acquire valid glort range */
 878         if (!fm10k_glort_valid(hw))
 879                 return;
 880
 881         if (dev->data->promiscuous) {
 882                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 883                         "since promisc mode is enabled");
 884                 return;
 885         }
 886
 887         fm10k_mbx_lock(hw);
 888         /* Change mode to unicast mode */
 889         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 890                                 FM10K_XCAST_MODE_NONE);
 891         fm10k_mbx_unlock(hw);
 892
 893         if (status != FM10K_SUCCESS)
 894                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 895 }
 896
 897 static void
 898 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 899 {
 900         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 901         uint32_t dglortdec, pool_len, rss_len, i;
 902         uint16_t nb_queue_pools;
 903         struct fm10k_macvlan_filter_info *macvlan;
 904
 905         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 906         nb_queue_pools = macvlan->nb_queue_pools;
 907         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 908         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 909         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 910
 911         /* Establish only MAP 0 as valid */
 912         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 913
 914         /* Configure VMDQ/RSS DGlort Decoder */
 915         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 916
 917         /* Invalidate all other GLORT entries */
 918         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 919                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 920                                 FM10K_DGLORTMAP_NONE);
 921 }
 922
 923 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 924 static int
 925 fm10k_dev_start(struct rte_eth_dev *dev)
 926 {
 927         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 928         int i, diag;
 929
 930         PMD_INIT_FUNC_TRACE();
 931
 932         /* stop, init, then start the hw */
 933         diag = fm10k_stop_hw(hw);
 934         if (diag != FM10K_SUCCESS) {
 935                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 936                 return -EIO;
 937         }
 938
 939         diag = fm10k_init_hw(hw);
 940         if (diag != FM10K_SUCCESS) {
 941                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 942                 return -EIO;
 943         }
 944
 945         diag = fm10k_start_hw(hw);
 946         if (diag != FM10K_SUCCESS) {
 947                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 948                 return -EIO;
 949         }
 950
 951         diag = fm10k_dev_tx_init(dev);
 952         if (diag) {
 953                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 954                 return diag;
 955         }
 956
 957         diag = fm10k_dev_rx_init(dev);
 958         if (diag) {
 959                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 960                 return diag;
 961         }
 962
 963         if (hw->mac.type == fm10k_mac_pf)
 964                 fm10k_dev_dglort_map_configure(dev);
 965
 966         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 967                 struct fm10k_rx_queue *rxq;
 968                 rxq = dev->data->rx_queues[i];
 969
 970                 if (rxq->rx_deferred_start)
 971                         continue;
 972                 diag = fm10k_dev_rx_queue_start(dev, i);
 973                 if (diag != 0) {
 974                         int j;
 975                         for (j = 0; j < i; ++j)
 976                                 rx_queue_clean(dev->data->rx_queues[j]);
 977                         return diag;
 978                 }
 979         }
 980
 981         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 982                 struct fm10k_tx_queue *txq;
 983                 txq = dev->data->tx_queues[i];
 984
 985                 if (txq->tx_deferred_start)
 986                         continue;
 987                 diag = fm10k_dev_tx_queue_start(dev, i);
 988                 if (diag != 0) {
 989                         int j;
 990                         for (j = 0; j < i; ++j)
 991                                 tx_queue_clean(dev->data->tx_queues[j]);
 992                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 993                                 rx_queue_clean(dev->data->rx_queues[j]);
 994                         return diag;
 995                 }
 996         }
 997
 998         /* Update default vlan when not in VMDQ mode */
 999         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1000                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1001
1002         return 0;
1003 }
1004
1005 static void
1006 fm10k_dev_stop(struct rte_eth_dev *dev)
1007 {
1008         int i;
1009
1010         PMD_INIT_FUNC_TRACE();
1011
1012         if (dev->data->tx_queues)
1013                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1014                         fm10k_dev_tx_queue_stop(dev, i);
1015
1016         if (dev->data->rx_queues)
1017                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1018                         fm10k_dev_rx_queue_stop(dev, i);
1019 }
1020
1021 static void
1022 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1023 {
1024         int i;
1025
1026         PMD_INIT_FUNC_TRACE();
1027
1028         if (dev->data->tx_queues) {
1029                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1030                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1031
1032                         txq->ops->release_mbufs(txq);
1033                 }
1034         }
1035
1036         if (dev->data->rx_queues) {
1037                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1038                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1039         }
1040 }
1041
1042 static void
1043 fm10k_dev_close(struct rte_eth_dev *dev)
1044 {
1045         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1046         uint16_t nb_lport;
1047         struct fm10k_macvlan_filter_info *macvlan;
1048
1049         PMD_INIT_FUNC_TRACE();
1050
1051         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1052         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1053         fm10k_mbx_lock(hw);
1054         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1055                 nb_lport, false);
1056         fm10k_mbx_unlock(hw);
1057
1058         /* Stop mailbox service first */
1059         fm10k_close_mbx_service(hw);
1060         fm10k_dev_stop(dev);
1061         fm10k_dev_queue_release(dev);
1062         fm10k_stop_hw(hw);
1063 }
1064
1065 static int
1066 fm10k_link_update(struct rte_eth_dev *dev,
1067         __rte_unused int wait_to_complete)
1068 {
1069         PMD_INIT_FUNC_TRACE();
1070
1071         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1072          * x8 PCIe interface. For now, we leave the speed undefined since there
1073          * is no 50Gbps Ethernet. */
1074         dev->data->dev_link.link_speed  = 0;
1075         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1076         dev->data->dev_link.link_status = 1;
1077
1078         return 0;
1079 }
1080
1081 static void
1082 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1083 {
1084         uint64_t ipackets, opackets, ibytes, obytes;
1085         struct fm10k_hw *hw =
1086                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1087         struct fm10k_hw_stats *hw_stats =
1088                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1089         int i;
1090
1091         PMD_INIT_FUNC_TRACE();
1092
1093         fm10k_update_hw_stats(hw, hw_stats);
1094
1095         ipackets = opackets = ibytes = obytes = 0;
1096         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1097                 (i < hw->mac.max_queues); ++i) {
1098                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1099                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1100                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1101                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1102                 ipackets += stats->q_ipackets[i];
1103                 opackets += stats->q_opackets[i];
1104                 ibytes   += stats->q_ibytes[i];
1105                 obytes   += stats->q_obytes[i];
1106         }
1107         stats->ipackets = ipackets;
1108         stats->opackets = opackets;
1109         stats->ibytes = ibytes;
1110         stats->obytes = obytes;
1111 }
1112
1113 static void
1114 fm10k_stats_reset(struct rte_eth_dev *dev)
1115 {
1116         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1117         struct fm10k_hw_stats *hw_stats =
1118                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1119
1120         PMD_INIT_FUNC_TRACE();
1121
1122         memset(hw_stats, 0, sizeof(*hw_stats));
1123         fm10k_rebind_hw_stats(hw, hw_stats);
1124 }
1125
1126 static void
1127 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1128         struct rte_eth_dev_info *dev_info)
1129 {
1130         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1131
1132         PMD_INIT_FUNC_TRACE();
1133
1134         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1135         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1136         dev_info->max_rx_queues      = hw->mac.max_queues;
1137         dev_info->max_tx_queues      = hw->mac.max_queues;
1138         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1139         dev_info->max_hash_mac_addrs = 0;
1140         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1141         dev_info->vmdq_pool_base     = 0;
1142         dev_info->vmdq_queue_base    = 0;
1143         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1144         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1145         dev_info->rx_offload_capa =
1146                 DEV_RX_OFFLOAD_VLAN_STRIP |
1147                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1148                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1149                 DEV_RX_OFFLOAD_TCP_CKSUM;
1150         dev_info->tx_offload_capa =
1151                 DEV_TX_OFFLOAD_VLAN_INSERT |
1152                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1153                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1154                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1155                 DEV_TX_OFFLOAD_TCP_TSO;
1156
1157         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1158         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1159
1160         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1161                 .rx_thresh = {
1162                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1163                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1164                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1165                 },
1166                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1167                 .rx_drop_en = 0,
1168         };
1169
1170         dev_info->default_txconf = (struct rte_eth_txconf) {
1171                 .tx_thresh = {
1172                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1173                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1174                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1175                 },
1176                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1177                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1178                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1179         };
1180
1181         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1182                 .nb_max = FM10K_MAX_RX_DESC,
1183                 .nb_min = FM10K_MIN_RX_DESC,
1184                 .nb_align = FM10K_MULT_RX_DESC,
1185         };
1186
1187         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1188                 .nb_max = FM10K_MAX_TX_DESC,
1189                 .nb_min = FM10K_MIN_TX_DESC,
1190                 .nb_align = FM10K_MULT_TX_DESC,
1191         };
1192 }
1193
1194 static int
1195 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1196 {
1197         s32 result;
1198         uint16_t mac_num = 0;
1199         uint32_t vid_idx, vid_bit, mac_index;
1200         struct fm10k_hw *hw;
1201         struct fm10k_macvlan_filter_info *macvlan;
1202         struct rte_eth_dev_data *data = dev->data;
1203
1204         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1205         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1206
1207         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1208                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1209                 return (-EINVAL);
1210         }
1211
1212         if (vlan_id > ETH_VLAN_ID_MAX) {
1213                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1214                 return (-EINVAL);
1215         }
1216
1217         vid_idx = FM10K_VFTA_IDX(vlan_id);
1218         vid_bit = FM10K_VFTA_BIT(vlan_id);
1219         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1220         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1221                 return 0;
1222         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1223         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1224                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1225                         "in the VLAN filter table");
1226                 return (-EINVAL);
1227         }
1228
1229         fm10k_mbx_lock(hw);
1230         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1231         fm10k_mbx_unlock(hw);
1232         if (result != FM10K_SUCCESS) {
1233                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1234                 return (-EIO);
1235         }
1236
1237         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1238                         (result == FM10K_SUCCESS); mac_index++) {
1239                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1240                         continue;
1241                 if (mac_num > macvlan->mac_num - 1) {
1242                         PMD_INIT_LOG(ERR, "MAC address number "
1243                                         "not match");
1244                         break;
1245                 }
1246                 fm10k_mbx_lock(hw);
1247                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1248                         data->mac_addrs[mac_index].addr_bytes,
1249                         vlan_id, on, 0);
1250                 fm10k_mbx_unlock(hw);
1251                 mac_num++;
1252         }
1253         if (result != FM10K_SUCCESS) {
1254                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1255                 return (-EIO);
1256         }
1257
1258         if (on) {
1259                 macvlan->vlan_num++;
1260                 macvlan->vfta[vid_idx] |= vid_bit;
1261         } else {
1262                 macvlan->vlan_num--;
1263                 macvlan->vfta[vid_idx] &= ~vid_bit;
1264         }
1265         return 0;
1266 }
1267
1268 static void
1269 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1270 {
1271         if (mask & ETH_VLAN_STRIP_MASK) {
1272                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1273                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1274                                         "always on in fm10k");
1275         }
1276
1277         if (mask & ETH_VLAN_EXTEND_MASK) {
1278                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1279                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1280                                         "supported in fm10k");
1281         }
1282
1283         if (mask & ETH_VLAN_FILTER_MASK) {
1284                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1285                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1286         }
1287 }
1288
1289 /* Add/Remove a MAC address, and update filters to main VSI */
1290 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1291                 const u8 *mac, bool add, uint32_t pool)
1292 {
1293         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1294         struct fm10k_macvlan_filter_info *macvlan;
1295         uint32_t i, j, k;
1296
1297         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1298
1299         if (pool != MAIN_VSI_POOL_NUMBER) {
1300                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1301                         "mac to pool %u", pool);
1302                 return;
1303         }
1304         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1305                 if (!macvlan->vfta[j])
1306                         continue;
1307                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1308                         if (!(macvlan->vfta[j] & (1 << k)))
1309                                 continue;
1310                         if (i + 1 > macvlan->vlan_num) {
1311                                 PMD_INIT_LOG(ERR, "vlan number not match");
1312                                 return;
1313                         }
1314                         fm10k_mbx_lock(hw);
1315                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1316                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1317                         fm10k_mbx_unlock(hw);
1318                         i++;
1319                 }
1320         }
1321 }
1322
1323 /* Add/Remove a MAC address, and update filters to VMDQ */
1324 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1325                 const u8 *mac, bool add, uint32_t pool)
1326 {
1327         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1328         struct fm10k_macvlan_filter_info *macvlan;
1329         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1330         uint32_t i;
1331
1332         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1333         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1334
1335         if (pool > macvlan->nb_queue_pools) {
1336                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1337                         " Max pool is %u",
1338                         pool, macvlan->nb_queue_pools);
1339                 return;
1340         }
1341         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1342                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1343                         continue;
1344                 fm10k_mbx_lock(hw);
1345                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1346                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1347                 fm10k_mbx_unlock(hw);
1348         }
1349 }
1350
1351 /* Add/Remove a MAC address, and update filters */
1352 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1353                 const u8 *mac, bool add, uint32_t pool)
1354 {
1355         struct fm10k_macvlan_filter_info *macvlan;
1356
1357         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1358
1359         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1360                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1361         else
1362                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1363
1364         if (add)
1365                 macvlan->mac_num++;
1366         else
1367                 macvlan->mac_num--;
1368 }
1369
1370 /* Add a MAC address, and update filters */
1371 static void
1372 fm10k_macaddr_add(struct rte_eth_dev *dev,
1373                 struct ether_addr *mac_addr,
1374                 uint32_t index,
1375                 uint32_t pool)
1376 {
1377         struct fm10k_macvlan_filter_info *macvlan;
1378
1379         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1380         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1381         macvlan->mac_vmdq_id[index] = pool;
1382 }
1383
1384 /* Remove a MAC address, and update filters */
1385 static void
1386 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1387 {
1388         struct rte_eth_dev_data *data = dev->data;
1389         struct fm10k_macvlan_filter_info *macvlan;
1390
1391         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1392         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1393                         FALSE, macvlan->mac_vmdq_id[index]);
1394         macvlan->mac_vmdq_id[index] = 0;
1395 }
1396
1397 static inline int
1398 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1399 {
1400         if ((request < min) || (request > max) || ((request % mult) != 0))
1401                 return -1;
1402         else
1403                 return 0;
1404 }
1405
1406 /*
1407  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1408  * the physical address is required. If the memzone is already created, then
1409  * this function returns a pointer to the existing memzone.
1410  */
1411 static inline const struct rte_memzone *
1412 allocate_hw_ring(const char *driver_name, const char *ring_name,
1413         uint8_t port_id, uint16_t queue_id, int socket_id,
1414         uint32_t size, uint32_t align)
1415 {
1416         char name[RTE_MEMZONE_NAMESIZE];
1417         const struct rte_memzone *mz;
1418
1419         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1420                  driver_name, ring_name, port_id, queue_id, socket_id);
1421
1422         /* return the memzone if it already exists */
1423         mz = rte_memzone_lookup(name);
1424         if (mz)
1425                 return mz;
1426
1427 #ifdef RTE_LIBRTE_XEN_DOM0
1428         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1429                                            RTE_PGSIZE_2M);
1430 #else
1431         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1432 #endif
1433 }
1434
1435 static inline int
1436 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1437 {
1438         if ((request < min) || (request > max) || ((div % request) != 0))
1439                 return -1;
1440         else
1441                 return 0;
1442 }
1443
1444 static inline int
1445 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1446 {
1447         uint16_t rx_free_thresh;
1448
1449         if (conf->rx_free_thresh == 0)
1450                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1451         else
1452                 rx_free_thresh = conf->rx_free_thresh;
1453
1454         /* make sure the requested threshold satisfies the constraints */
1455         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1456                         FM10K_RX_FREE_THRESH_MAX(q),
1457                         FM10K_RX_FREE_THRESH_DIV(q),
1458                         rx_free_thresh)) {
1459                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1460                         "less than or equal to %u, "
1461                         "greater than or equal to %u, "
1462                         "and a divisor of %u",
1463                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1464                         FM10K_RX_FREE_THRESH_MIN(q),
1465                         FM10K_RX_FREE_THRESH_DIV(q));
1466                 return (-EINVAL);
1467         }
1468
1469         q->alloc_thresh = rx_free_thresh;
1470         q->drop_en = conf->rx_drop_en;
1471         q->rx_deferred_start = conf->rx_deferred_start;
1472
1473         return 0;
1474 }
1475
1476 /*
1477  * Hardware requires specific alignment for Rx packet buffers. At
1478  * least one of the following two conditions must be satisfied.
1479  *  1. Address is 512B aligned
1480  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1481  *
1482  * As such, the driver may need to adjust the DMA address within the
1483  * buffer by up to 512B.
1484  *
1485  * return 1 if the element size is valid, otherwise return 0.
1486  */
1487 static int
1488 mempool_element_size_valid(struct rte_mempool *mp)
1489 {
1490         uint32_t min_size;
1491
1492         /* elt_size includes mbuf header and headroom */
1493         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1494                         RTE_PKTMBUF_HEADROOM;
1495
1496         /* account for up to 512B of alignment */
1497         min_size -= FM10K_RX_DATABUF_ALIGN;
1498
1499         /* sanity check for overflow */
1500         if (min_size > mp->elt_size)
1501                 return 0;
1502
1503         /* size is valid */
1504         return 1;
1505 }
1506
1507 static int
1508 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1509         uint16_t nb_desc, unsigned int socket_id,
1510         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1511 {
1512         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1513         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1514         struct fm10k_rx_queue *q;
1515         const struct rte_memzone *mz;
1516
1517         PMD_INIT_FUNC_TRACE();
1518
1519         /* make sure the mempool element size can account for alignment. */
1520         if (!mempool_element_size_valid(mp)) {
1521                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1522                 return (-EINVAL);
1523         }
1524
1525         /* make sure a valid number of descriptors have been requested */
1526         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1527                                 FM10K_MULT_RX_DESC, nb_desc)) {
1528                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1529                         "less than or equal to %"PRIu32", "
1530                         "greater than or equal to %u, "
1531                         "and a multiple of %u",
1532                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1533                         FM10K_MULT_RX_DESC);
1534                 return (-EINVAL);
1535         }
1536
1537         /*
1538          * if this queue existed already, free the associated memory. The
1539          * queue cannot be reused in case we need to allocate memory on
1540          * different socket than was previously used.
1541          */
1542         if (dev->data->rx_queues[queue_id] != NULL) {
1543                 rx_queue_free(dev->data->rx_queues[queue_id]);
1544                 dev->data->rx_queues[queue_id] = NULL;
1545         }
1546
1547         /* allocate memory for the queue structure */
1548         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1549                                 socket_id);
1550         if (q == NULL) {
1551                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1552                 return (-ENOMEM);
1553         }
1554
1555         /* setup queue */
1556         q->mp = mp;
1557         q->nb_desc = nb_desc;
1558         q->nb_fake_desc = FM10K_MULT_RX_DESC;
1559         q->port_id = dev->data->port_id;
1560         q->queue_id = queue_id;
1561         q->tail_ptr = (volatile uint32_t *)
1562                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1563         if (handle_rxconf(q, conf))
1564                 return (-EINVAL);
1565
1566         /* allocate memory for the software ring */
1567         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1568                         (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1569                         RTE_CACHE_LINE_SIZE, socket_id);
1570         if (q->sw_ring == NULL) {
1571                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1572                 rte_free(q);
1573                 return (-ENOMEM);
1574         }
1575
1576         /*
1577          * allocate memory for the hardware descriptor ring. A memzone large
1578          * enough to hold the maximum ring size is requested to allow for
1579          * resizing in later calls to the queue setup function.
1580          */
1581         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1582                                 dev->data->port_id, queue_id, socket_id,
1583                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1584         if (mz == NULL) {
1585                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1586                 rte_free(q->sw_ring);
1587                 rte_free(q);
1588                 return (-ENOMEM);
1589         }
1590         q->hw_ring = mz->addr;
1591 #ifdef RTE_LIBRTE_XEN_DOM0
1592         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1593 #else
1594         q->hw_ring_phys_addr = mz->phys_addr;
1595 #endif
1596
1597         /* Check if number of descs satisfied Vector requirement */
1598         if (!rte_is_power_of_2(nb_desc)) {
1599                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1600                                     "preconditions - canceling the feature for "
1601                                     "the whole port[%d]",
1602                              q->queue_id, q->port_id);
1603                 dev_info->rx_vec_allowed = false;
1604         } else
1605                 fm10k_rxq_vec_setup(q);
1606
1607         dev->data->rx_queues[queue_id] = q;
1608         return 0;
1609 }
1610
1611 static void
1612 fm10k_rx_queue_release(void *queue)
1613 {
1614         PMD_INIT_FUNC_TRACE();
1615
1616         rx_queue_free(queue);
1617 }
1618
1619 static inline int
1620 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1621 {
1622         uint16_t tx_free_thresh;
1623         uint16_t tx_rs_thresh;
1624
1625         /* constraint MACROs require that tx_free_thresh is configured
1626          * before tx_rs_thresh */
1627         if (conf->tx_free_thresh == 0)
1628                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1629         else
1630                 tx_free_thresh = conf->tx_free_thresh;
1631
1632         /* make sure the requested threshold satisfies the constraints */
1633         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1634                         FM10K_TX_FREE_THRESH_MAX(q),
1635                         FM10K_TX_FREE_THRESH_DIV(q),
1636                         tx_free_thresh)) {
1637                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1638                         "less than or equal to %u, "
1639                         "greater than or equal to %u, "
1640                         "and a divisor of %u",
1641                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1642                         FM10K_TX_FREE_THRESH_MIN(q),
1643                         FM10K_TX_FREE_THRESH_DIV(q));
1644                 return (-EINVAL);
1645         }
1646
1647         q->free_thresh = tx_free_thresh;
1648
1649         if (conf->tx_rs_thresh == 0)
1650                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1651         else
1652                 tx_rs_thresh = conf->tx_rs_thresh;
1653
1654         q->tx_deferred_start = conf->tx_deferred_start;
1655
1656         /* make sure the requested threshold satisfies the constraints */
1657         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1658                         FM10K_TX_RS_THRESH_MAX(q),
1659                         FM10K_TX_RS_THRESH_DIV(q),
1660                         tx_rs_thresh)) {
1661                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1662                         "less than or equal to %u, "
1663                         "greater than or equal to %u, "
1664                         "and a divisor of %u",
1665                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1666                         FM10K_TX_RS_THRESH_MIN(q),
1667                         FM10K_TX_RS_THRESH_DIV(q));
1668                 return (-EINVAL);
1669         }
1670
1671         q->rs_thresh = tx_rs_thresh;
1672
1673         return 0;
1674 }
1675
1676 static int
1677 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1678         uint16_t nb_desc, unsigned int socket_id,
1679         const struct rte_eth_txconf *conf)
1680 {
1681         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1682         struct fm10k_tx_queue *q;
1683         const struct rte_memzone *mz;
1684
1685         PMD_INIT_FUNC_TRACE();
1686
1687         /* make sure a valid number of descriptors have been requested */
1688         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1689                                 FM10K_MULT_TX_DESC, nb_desc)) {
1690                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1691                         "less than or equal to %"PRIu32", "
1692                         "greater than or equal to %u, "
1693                         "and a multiple of %u",
1694                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1695                         FM10K_MULT_TX_DESC);
1696                 return (-EINVAL);
1697         }
1698
1699         /*
1700          * if this queue existed already, free the associated memory. The
1701          * queue cannot be reused in case we need to allocate memory on
1702          * different socket than was previously used.
1703          */
1704         if (dev->data->tx_queues[queue_id] != NULL) {
1705                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1706
1707                 txq->ops->release_mbufs(txq);
1708                 dev->data->tx_queues[queue_id] = NULL;
1709         }
1710
1711         /* allocate memory for the queue structure */
1712         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1713                                 socket_id);
1714         if (q == NULL) {
1715                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1716                 return (-ENOMEM);
1717         }
1718
1719         /* setup queue */
1720         q->nb_desc = nb_desc;
1721         q->port_id = dev->data->port_id;
1722         q->queue_id = queue_id;
1723         q->txq_flags = conf->txq_flags;
1724         q->ops = &def_txq_ops;
1725         q->tail_ptr = (volatile uint32_t *)
1726                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1727         if (handle_txconf(q, conf))
1728                 return (-EINVAL);
1729
1730         /* allocate memory for the software ring */
1731         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1732                                         nb_desc * sizeof(struct rte_mbuf *),
1733                                         RTE_CACHE_LINE_SIZE, socket_id);
1734         if (q->sw_ring == NULL) {
1735                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1736                 rte_free(q);
1737                 return (-ENOMEM);
1738         }
1739
1740         /*
1741          * allocate memory for the hardware descriptor ring. A memzone large
1742          * enough to hold the maximum ring size is requested to allow for
1743          * resizing in later calls to the queue setup function.
1744          */
1745         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1746                                 dev->data->port_id, queue_id, socket_id,
1747                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1748         if (mz == NULL) {
1749                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1750                 rte_free(q->sw_ring);
1751                 rte_free(q);
1752                 return (-ENOMEM);
1753         }
1754         q->hw_ring = mz->addr;
1755 #ifdef RTE_LIBRTE_XEN_DOM0
1756         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1757 #else
1758         q->hw_ring_phys_addr = mz->phys_addr;
1759 #endif
1760
1761         /*
1762          * allocate memory for the RS bit tracker. Enough slots to hold the
1763          * descriptor index for each RS bit needing to be set are required.
1764          */
1765         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1766                                 ((nb_desc + 1) / q->rs_thresh) *
1767                                 sizeof(uint16_t),
1768                                 RTE_CACHE_LINE_SIZE, socket_id);
1769         if (q->rs_tracker.list == NULL) {
1770                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1771                 rte_free(q->sw_ring);
1772                 rte_free(q);
1773                 return (-ENOMEM);
1774         }
1775
1776         dev->data->tx_queues[queue_id] = q;
1777         return 0;
1778 }
1779
1780 static void
1781 fm10k_tx_queue_release(void *queue)
1782 {
1783         struct fm10k_tx_queue *q = queue;
1784         PMD_INIT_FUNC_TRACE();
1785
1786         q->ops->release_mbufs(q);
1787 }
1788
1789 static int
1790 fm10k_reta_update(struct rte_eth_dev *dev,
1791                         struct rte_eth_rss_reta_entry64 *reta_conf,
1792                         uint16_t reta_size)
1793 {
1794         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1795         uint16_t i, j, idx, shift;
1796         uint8_t mask;
1797         uint32_t reta;
1798
1799         PMD_INIT_FUNC_TRACE();
1800
1801         if (reta_size > FM10K_MAX_RSS_INDICES) {
1802                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1803                         "(%d) doesn't match the number hardware can supported "
1804                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1805                 return -EINVAL;
1806         }
1807
1808         /*
1809          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1810          * 128-entries in 32 registers
1811          */
1812         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1813                 idx = i / RTE_RETA_GROUP_SIZE;
1814                 shift = i % RTE_RETA_GROUP_SIZE;
1815                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1816                                 BIT_MASK_PER_UINT32);
1817                 if (mask == 0)
1818                         continue;
1819
1820                 reta = 0;
1821                 if (mask != BIT_MASK_PER_UINT32)
1822                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1823
1824                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1825                         if (mask & (0x1 << j)) {
1826                                 if (mask != 0xF)
1827                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1828                                 reta |= reta_conf[idx].reta[shift + j] <<
1829                                                 (CHAR_BIT * j);
1830                         }
1831                 }
1832                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1833         }
1834
1835         return 0;
1836 }
1837
1838 static int
1839 fm10k_reta_query(struct rte_eth_dev *dev,
1840                         struct rte_eth_rss_reta_entry64 *reta_conf,
1841                         uint16_t reta_size)
1842 {
1843         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1844         uint16_t i, j, idx, shift;
1845         uint8_t mask;
1846         uint32_t reta;
1847
1848         PMD_INIT_FUNC_TRACE();
1849
1850         if (reta_size < FM10K_MAX_RSS_INDICES) {
1851                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1852                         "(%d) doesn't match the number hardware can supported "
1853                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1854                 return -EINVAL;
1855         }
1856
1857         /*
1858          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1859          * 128-entries in 32 registers
1860          */
1861         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1862                 idx = i / RTE_RETA_GROUP_SIZE;
1863                 shift = i % RTE_RETA_GROUP_SIZE;
1864                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1865                                 BIT_MASK_PER_UINT32);
1866                 if (mask == 0)
1867                         continue;
1868
1869                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1870                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1871                         if (mask & (0x1 << j))
1872                                 reta_conf[idx].reta[shift + j] = ((reta >>
1873                                         CHAR_BIT * j) & UINT8_MAX);
1874                 }
1875         }
1876
1877         return 0;
1878 }
1879
1880 static int
1881 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1882         struct rte_eth_rss_conf *rss_conf)
1883 {
1884         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1885         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1886         uint32_t mrqc;
1887         uint64_t hf = rss_conf->rss_hf;
1888         int i;
1889
1890         PMD_INIT_FUNC_TRACE();
1891
1892         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1893                 FM10K_RSSRK_ENTRIES_PER_REG)
1894                 return -EINVAL;
1895
1896         if (hf == 0)
1897                 return -EINVAL;
1898
1899         mrqc = 0;
1900         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1901         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1902         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1903         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1904         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1905         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1906         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1907         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1908         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1909
1910         /* If the mapping doesn't fit any supported, return */
1911         if (mrqc == 0)
1912                 return -EINVAL;
1913
1914         if (key != NULL)
1915                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1916                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1917
1918         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1919
1920         return 0;
1921 }
1922
1923 static int
1924 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1925         struct rte_eth_rss_conf *rss_conf)
1926 {
1927         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1928         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1929         uint32_t mrqc;
1930         uint64_t hf;
1931         int i;
1932
1933         PMD_INIT_FUNC_TRACE();
1934
1935         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1936                                 FM10K_RSSRK_ENTRIES_PER_REG)
1937                 return -EINVAL;
1938
1939         if (key != NULL)
1940                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1941                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1942
1943         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1944         hf = 0;
1945         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1946         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1947         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1948         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1949         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1950         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1951         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1952         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1953         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1954
1955         rss_conf->rss_hf = hf;
1956
1957         return 0;
1958 }
1959
1960 static void
1961 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1962 {
1963         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1964         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1965
1966         /* Bind all local non-queue interrupt to vector 0 */
1967         int_map |= 0;
1968
1969         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1970         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1971         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1972         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1973         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1974         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1975
1976         /* Enable misc causes */
1977         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1978                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1979                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1980                                 FM10K_EIMR_ENABLE(MAILBOX) |
1981                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1982                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1983                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1984                                 FM10K_EIMR_ENABLE(VFLR));
1985
1986         /* Enable ITR 0 */
1987         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1988                                         FM10K_ITR_MASK_CLEAR);
1989         FM10K_WRITE_FLUSH(hw);
1990 }
1991
1992 static void
1993 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1994 {
1995         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1996         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1997
1998         int_map |= 0;
1999
2000         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
2001         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
2002         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
2003         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
2004         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
2005         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
2006
2007         /* Disable misc causes */
2008         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2009                                 FM10K_EIMR_DISABLE(THI_FAULT) |
2010                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
2011                                 FM10K_EIMR_DISABLE(MAILBOX) |
2012                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
2013                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2014                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2015                                 FM10K_EIMR_DISABLE(VFLR));
2016
2017         /* Disable ITR 0 */
2018         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2019         FM10K_WRITE_FLUSH(hw);
2020 }
2021
2022 static void
2023 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2024 {
2025         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2026         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2027
2028         /* Bind all local non-queue interrupt to vector 0 */
2029         int_map |= 0;
2030
2031         /* Only INT 0 available, other 15 are reserved. */
2032         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2033
2034         /* Enable ITR 0 */
2035         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2036                                         FM10K_ITR_MASK_CLEAR);
2037         FM10K_WRITE_FLUSH(hw);
2038 }
2039
2040 static void
2041 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2042 {
2043         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2044         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2045
2046         int_map |= 0;
2047
2048         /* Only INT 0 available, other 15 are reserved. */
2049         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2050
2051         /* Disable ITR 0 */
2052         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2053         FM10K_WRITE_FLUSH(hw);
2054 }
2055
2056 static int
2057 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2058 {
2059         struct fm10k_fault fault;
2060         int err;
2061         const char *estr = "Unknown error";
2062
2063         /* Process PCA fault */
2064         if (eicr & FM10K_EICR_PCA_FAULT) {
2065                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2066                 if (err)
2067                         goto error;
2068                 switch (fault.type) {
2069                 case PCA_NO_FAULT:
2070                         estr = "PCA_NO_FAULT"; break;
2071                 case PCA_UNMAPPED_ADDR:
2072                         estr = "PCA_UNMAPPED_ADDR"; break;
2073                 case PCA_BAD_QACCESS_PF:
2074                         estr = "PCA_BAD_QACCESS_PF"; break;
2075                 case PCA_BAD_QACCESS_VF:
2076                         estr = "PCA_BAD_QACCESS_VF"; break;
2077                 case PCA_MALICIOUS_REQ:
2078                         estr = "PCA_MALICIOUS_REQ"; break;
2079                 case PCA_POISONED_TLP:
2080                         estr = "PCA_POISONED_TLP"; break;
2081                 case PCA_TLP_ABORT:
2082                         estr = "PCA_TLP_ABORT"; break;
2083                 default:
2084                         goto error;
2085                 }
2086                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2087                         estr, fault.func ? "VF" : "PF", fault.func,
2088                         fault.address, fault.specinfo);
2089         }
2090
2091         /* Process THI fault */
2092         if (eicr & FM10K_EICR_THI_FAULT) {
2093                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2094                 if (err)
2095                         goto error;
2096                 switch (fault.type) {
2097                 case THI_NO_FAULT:
2098                         estr = "THI_NO_FAULT"; break;
2099                 case THI_MAL_DIS_Q_FAULT:
2100                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2101                 default:
2102                         goto error;
2103                 }
2104                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2105                         estr, fault.func ? "VF" : "PF", fault.func,
2106                         fault.address, fault.specinfo);
2107         }
2108
2109         /* Process FUM fault */
2110         if (eicr & FM10K_EICR_FUM_FAULT) {
2111                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2112                 if (err)
2113                         goto error;
2114                 switch (fault.type) {
2115                 case FUM_NO_FAULT:
2116                         estr = "FUM_NO_FAULT"; break;
2117                 case FUM_UNMAPPED_ADDR:
2118                         estr = "FUM_UNMAPPED_ADDR"; break;
2119                 case FUM_POISONED_TLP:
2120                         estr = "FUM_POISONED_TLP"; break;
2121                 case FUM_BAD_VF_QACCESS:
2122                         estr = "FUM_BAD_VF_QACCESS"; break;
2123                 case FUM_ADD_DECODE_ERR:
2124                         estr = "FUM_ADD_DECODE_ERR"; break;
2125                 case FUM_RO_ERROR:
2126                         estr = "FUM_RO_ERROR"; break;
2127                 case FUM_QPRC_CRC_ERROR:
2128                         estr = "FUM_QPRC_CRC_ERROR"; break;
2129                 case FUM_CSR_TIMEOUT:
2130                         estr = "FUM_CSR_TIMEOUT"; break;
2131                 case FUM_INVALID_TYPE:
2132                         estr = "FUM_INVALID_TYPE"; break;
2133                 case FUM_INVALID_LENGTH:
2134                         estr = "FUM_INVALID_LENGTH"; break;
2135                 case FUM_INVALID_BE:
2136                         estr = "FUM_INVALID_BE"; break;
2137                 case FUM_INVALID_ALIGN:
2138                         estr = "FUM_INVALID_ALIGN"; break;
2139                 default:
2140                         goto error;
2141                 }
2142                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2143                         estr, fault.func ? "VF" : "PF", fault.func,
2144                         fault.address, fault.specinfo);
2145         }
2146
2147         return 0;
2148 error:
2149         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2150         return err;
2151 }
2152
2153 /**
2154  * PF interrupt handler triggered by NIC for handling specific interrupt.
2155  *
2156  * @param handle
2157  *  Pointer to interrupt handle.
2158  * @param param
2159  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2160  *
2161  * @return
2162  *  void
2163  */
2164 static void
2165 fm10k_dev_interrupt_handler_pf(
2166                         __rte_unused struct rte_intr_handle *handle,
2167                         void *param)
2168 {
2169         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2170         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2171         uint32_t cause, status;
2172
2173         if (hw->mac.type != fm10k_mac_pf)
2174                 return;
2175
2176         cause = FM10K_READ_REG(hw, FM10K_EICR);
2177
2178         /* Handle PCI fault cases */
2179         if (cause & FM10K_EICR_FAULT_MASK) {
2180                 PMD_INIT_LOG(ERR, "INT: find fault!");
2181                 fm10k_dev_handle_fault(hw, cause);
2182         }
2183
2184         /* Handle switch up/down */
2185         if (cause & FM10K_EICR_SWITCHNOTREADY)
2186                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2187
2188         if (cause & FM10K_EICR_SWITCHREADY)
2189                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2190
2191         /* Handle mailbox message */
2192         fm10k_mbx_lock(hw);
2193         hw->mbx.ops.process(hw, &hw->mbx);
2194         fm10k_mbx_unlock(hw);
2195
2196         /* Handle SRAM error */
2197         if (cause & FM10K_EICR_SRAMERROR) {
2198                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2199
2200                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2201                 /* Write to clear pending bits */
2202                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2203
2204                 /* Todo: print out error message after shared code  updates */
2205         }
2206
2207         /* Clear these 3 events if having any */
2208         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2209                  FM10K_EICR_SWITCHREADY;
2210         if (cause)
2211                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2212
2213         /* Re-enable interrupt from device side */
2214         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2215                                         FM10K_ITR_MASK_CLEAR);
2216         /* Re-enable interrupt from host side */
2217         rte_intr_enable(&(dev->pci_dev->intr_handle));
2218 }
2219
2220 /**
2221  * VF interrupt handler triggered by NIC for handling specific interrupt.
2222  *
2223  * @param handle
2224  *  Pointer to interrupt handle.
2225  * @param param
2226  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2227  *
2228  * @return
2229  *  void
2230  */
2231 static void
2232 fm10k_dev_interrupt_handler_vf(
2233                         __rte_unused struct rte_intr_handle *handle,
2234                         void *param)
2235 {
2236         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2237         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2238
2239         if (hw->mac.type != fm10k_mac_vf)
2240                 return;
2241
2242         /* Handle mailbox message if lock is acquired */
2243         fm10k_mbx_lock(hw);
2244         hw->mbx.ops.process(hw, &hw->mbx);
2245         fm10k_mbx_unlock(hw);
2246
2247         /* Re-enable interrupt from device side */
2248         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2249                                         FM10K_ITR_MASK_CLEAR);
2250         /* Re-enable interrupt from host side */
2251         rte_intr_enable(&(dev->pci_dev->intr_handle));
2252 }
2253
2254 /* Mailbox message handler in VF */
2255 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2256         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2257         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2258         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2259         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2260 };
2261
2262 /* Mailbox message handler in PF */
2263 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2264         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2265         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2266         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2267         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2268         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2269         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2270         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2271 };
2272
2273 static int
2274 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2275 {
2276         int err;
2277
2278         /* Initialize mailbox lock */
2279         fm10k_mbx_initlock(hw);
2280
2281         /* Replace default message handler with new ones */
2282         if (hw->mac.type == fm10k_mac_pf)
2283                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2284         else
2285                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2286
2287         if (err) {
2288                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2289                                 err);
2290                 return err;
2291         }
2292         /* Connect to SM for PF device or PF for VF device */
2293         return hw->mbx.ops.connect(hw, &hw->mbx);
2294 }
2295
2296 static void
2297 fm10k_close_mbx_service(struct fm10k_hw *hw)
2298 {
2299         /* Disconnect from SM for PF device or PF for VF device */
2300         hw->mbx.ops.disconnect(hw, &hw->mbx);
2301 }
2302
2303 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2304         .dev_configure          = fm10k_dev_configure,
2305         .dev_start              = fm10k_dev_start,
2306         .dev_stop               = fm10k_dev_stop,
2307         .dev_close              = fm10k_dev_close,
2308         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2309         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2310         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2311         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2312         .stats_get              = fm10k_stats_get,
2313         .stats_reset            = fm10k_stats_reset,
2314         .link_update            = fm10k_link_update,
2315         .dev_infos_get          = fm10k_dev_infos_get,
2316         .vlan_filter_set        = fm10k_vlan_filter_set,
2317         .vlan_offload_set       = fm10k_vlan_offload_set,
2318         .mac_addr_add           = fm10k_macaddr_add,
2319         .mac_addr_remove        = fm10k_macaddr_remove,
2320         .rx_queue_start         = fm10k_dev_rx_queue_start,
2321         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2322         .tx_queue_start         = fm10k_dev_tx_queue_start,
2323         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2324         .rx_queue_setup         = fm10k_rx_queue_setup,
2325         .rx_queue_release       = fm10k_rx_queue_release,
2326         .tx_queue_setup         = fm10k_tx_queue_setup,
2327         .tx_queue_release       = fm10k_tx_queue_release,
2328         .reta_update            = fm10k_reta_update,
2329         .reta_query             = fm10k_reta_query,
2330         .rss_hash_update        = fm10k_rss_hash_update,
2331         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2332 };
2333
2334 static void __attribute__((cold))
2335 fm10k_set_tx_function(struct rte_eth_dev *dev)
2336 {
2337         struct fm10k_tx_queue *txq;
2338         int i;
2339         int use_sse = 1;
2340
2341         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2342                 txq = dev->data->tx_queues[i];
2343                 if ((txq->txq_flags & FM10K_SIMPLE_TX_FLAG) !=
2344                         FM10K_SIMPLE_TX_FLAG) {
2345                         use_sse = 0;
2346                         break;
2347                 }
2348         }
2349
2350         if (use_sse) {
2351                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2352                         txq = dev->data->tx_queues[i];
2353                         fm10k_txq_vec_setup(txq);
2354                 }
2355                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2356         } else
2357                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2358 }
2359
2360 static void __attribute__((cold))
2361 fm10k_set_rx_function(struct rte_eth_dev *dev)
2362 {
2363         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2364         uint16_t i, rx_using_sse;
2365
2366         /* In order to allow Vector Rx there are a few configuration
2367          * conditions to be met.
2368          */
2369         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2370                 if (dev->data->scattered_rx)
2371                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2372                 else
2373                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2374         } else if (dev->data->scattered_rx)
2375                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2376
2377         rx_using_sse =
2378                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2379                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2380
2381         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2382                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2383
2384                 rxq->rx_using_sse = rx_using_sse;
2385         }
2386 }
2387
2388 static void
2389 fm10k_params_init(struct rte_eth_dev *dev)
2390 {
2391         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2392         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2393
2394         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2395          * there is no way to get link status without reading BAR4.  Until this
2396          * works, assume we have maximum bandwidth.
2397          * @todo - fix bus info
2398          */
2399         hw->bus_caps.speed = fm10k_bus_speed_8000;
2400         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2401         hw->bus_caps.payload = fm10k_bus_payload_512;
2402         hw->bus.speed = fm10k_bus_speed_8000;
2403         hw->bus.width = fm10k_bus_width_pcie_x8;
2404         hw->bus.payload = fm10k_bus_payload_256;
2405
2406         info->rx_vec_allowed = true;
2407 }
2408
2409 static int
2410 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2411 {
2412         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2413         int diag;
2414         struct fm10k_macvlan_filter_info *macvlan;
2415
2416         PMD_INIT_FUNC_TRACE();
2417
2418         dev->dev_ops = &fm10k_eth_dev_ops;
2419         dev->rx_pkt_burst = &fm10k_recv_pkts;
2420         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2421
2422         /* only initialize in the primary process */
2423         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2424                 return 0;
2425
2426         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2427         memset(macvlan, 0, sizeof(*macvlan));
2428         /* Vendor and Device ID need to be set before init of shared code */
2429         memset(hw, 0, sizeof(*hw));
2430         hw->device_id = dev->pci_dev->id.device_id;
2431         hw->vendor_id = dev->pci_dev->id.vendor_id;
2432         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2433         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2434         hw->revision_id = 0;
2435         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2436         if (hw->hw_addr == NULL) {
2437                 PMD_INIT_LOG(ERR, "Bad mem resource."
2438                         " Try to blacklist unused devices.");
2439                 return -EIO;
2440         }
2441
2442         /* Store fm10k_adapter pointer */
2443         hw->back = dev->data->dev_private;
2444
2445         /* Initialize the shared code */
2446         diag = fm10k_init_shared_code(hw);
2447         if (diag != FM10K_SUCCESS) {
2448                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2449                 return -EIO;
2450         }
2451
2452         /* Initialize parameters */
2453         fm10k_params_init(dev);
2454
2455         /* Initialize the hw */
2456         diag = fm10k_init_hw(hw);
2457         if (diag != FM10K_SUCCESS) {
2458                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2459                 return -EIO;
2460         }
2461
2462         /* Initialize MAC address(es) */
2463         dev->data->mac_addrs = rte_zmalloc("fm10k",
2464                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2465         if (dev->data->mac_addrs == NULL) {
2466                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2467                 return -ENOMEM;
2468         }
2469
2470         diag = fm10k_read_mac_addr(hw);
2471
2472         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2473                         &dev->data->mac_addrs[0]);
2474
2475         if (diag != FM10K_SUCCESS ||
2476                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2477
2478                 /* Generate a random addr */
2479                 eth_random_addr(hw->mac.addr);
2480                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2481                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2482                 &dev->data->mac_addrs[0]);
2483         }
2484
2485         /* Reset the hw statistics */
2486         fm10k_stats_reset(dev);
2487
2488         /* Reset the hw */
2489         diag = fm10k_reset_hw(hw);
2490         if (diag != FM10K_SUCCESS) {
2491                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2492                 return -EIO;
2493         }
2494
2495         /* Setup mailbox service */
2496         diag = fm10k_setup_mbx_service(hw);
2497         if (diag != FM10K_SUCCESS) {
2498                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2499                 return -EIO;
2500         }
2501
2502         /*PF/VF has different interrupt handling mechanism */
2503         if (hw->mac.type == fm10k_mac_pf) {
2504                 /* register callback func to eal lib */
2505                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2506                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2507
2508                 /* enable MISC interrupt */
2509                 fm10k_dev_enable_intr_pf(dev);
2510         } else { /* VF */
2511                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2512                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2513
2514                 fm10k_dev_enable_intr_vf(dev);
2515         }
2516
2517         /* Enable uio intr after callback registered */
2518         rte_intr_enable(&(dev->pci_dev->intr_handle));
2519
2520         hw->mac.ops.update_int_moderator(hw);
2521
2522         /* Make sure Switch Manager is ready before going forward. */
2523         if (hw->mac.type == fm10k_mac_pf) {
2524                 int switch_ready = 0;
2525                 int i;
2526
2527                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2528                         fm10k_mbx_lock(hw);
2529                         hw->mac.ops.get_host_state(hw, &switch_ready);
2530                         fm10k_mbx_unlock(hw);
2531                         if (switch_ready)
2532                                 break;
2533                         /* Delay some time to acquire async LPORT_MAP info. */
2534                         rte_delay_us(WAIT_SWITCH_MSG_US);
2535                 }
2536
2537                 if (switch_ready == 0) {
2538                         PMD_INIT_LOG(ERR, "switch is not ready");
2539                         return -1;
2540                 }
2541         }
2542
2543         /*
2544          * Below function will trigger operations on mailbox, acquire lock to
2545          * avoid race condition from interrupt handler. Operations on mailbox
2546          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2547          * will handle and generate an interrupt to our side. Then,  FIFO in
2548          * mailbox will be touched.
2549          */
2550         fm10k_mbx_lock(hw);
2551         /* Enable port first */
2552         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2553
2554         /* Set unicast mode by default. App can change to other mode in other
2555          * API func.
2556          */
2557         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2558                                         FM10K_XCAST_MODE_NONE);
2559
2560         fm10k_mbx_unlock(hw);
2561
2562         /* Add default mac address */
2563         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2564                 MAIN_VSI_POOL_NUMBER);
2565
2566         return 0;
2567 }
2568
2569 static int
2570 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2571 {
2572         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2573
2574         PMD_INIT_FUNC_TRACE();
2575
2576         /* only uninitialize in the primary process */
2577         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2578                 return 0;
2579
2580         /* safe to close dev here */
2581         fm10k_dev_close(dev);
2582
2583         dev->dev_ops = NULL;
2584         dev->rx_pkt_burst = NULL;
2585         dev->tx_pkt_burst = NULL;
2586
2587         /* disable uio/vfio intr */
2588         rte_intr_disable(&(dev->pci_dev->intr_handle));
2589
2590         /*PF/VF has different interrupt handling mechanism */
2591         if (hw->mac.type == fm10k_mac_pf) {
2592                 /* disable interrupt */
2593                 fm10k_dev_disable_intr_pf(dev);
2594
2595                 /* unregister callback func to eal lib */
2596                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2597                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2598         } else {
2599                 /* disable interrupt */
2600                 fm10k_dev_disable_intr_vf(dev);
2601
2602                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2603                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2604         }
2605
2606         /* free mac memory */
2607         if (dev->data->mac_addrs) {
2608                 rte_free(dev->data->mac_addrs);
2609                 dev->data->mac_addrs = NULL;
2610         }
2611
2612         memset(hw, 0, sizeof(*hw));
2613
2614         return 0;
2615 }
2616
2617 /*
2618  * The set of PCI devices this driver supports. This driver will enable both PF
2619  * and SRIOV-VF devices.
2620  */
2621 static const struct rte_pci_id pci_id_fm10k_map[] = {
2622 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2623 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2624 #include "rte_pci_dev_ids.h"
2625         { .vendor_id = 0, /* sentinel */ },
2626 };
2627
2628 static struct eth_driver rte_pmd_fm10k = {
2629         .pci_drv = {
2630                 .name = "rte_pmd_fm10k",
2631                 .id_table = pci_id_fm10k_map,
2632                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2633         },
2634         .eth_dev_init = eth_fm10k_dev_init,
2635         .eth_dev_uninit = eth_fm10k_dev_uninit,
2636         .dev_private_size = sizeof(struct fm10k_adapter),
2637 };
2638
2639 /*
2640  * Driver initialization routine.
2641  * Invoked once at EAL init time.
2642  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2643  */
2644 static int
2645 rte_pmd_fm10k_init(__rte_unused const char *name,
2646         __rte_unused const char *params)
2647 {
2648         PMD_INIT_FUNC_TRACE();
2649         rte_eth_driver_register(&rte_pmd_fm10k);
2650         return 0;
2651 }
2652
2653 static struct rte_driver rte_fm10k_driver = {
2654         .type = PMD_PDEV,
2655         .init = rte_pmd_fm10k_init,
2656 };
2657
2658 PMD_REGISTER_DRIVER(rte_fm10k_driver);