drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 #define FM10K_SIMPLE_TX_FLAG ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
  59                                 ETH_TXQ_FLAGS_NOOFFLOADS)
  60
  61 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  62 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  63 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  64 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  65 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  66 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  67 static int
  68 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  69 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  70         const u8 *mac, bool add, uint32_t pool);
  71 static void fm10k_tx_queue_release(void *queue);
  72 static void fm10k_rx_queue_release(void *queue);
  73 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  74 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
  75
  76 static void
  77 fm10k_mbx_initlock(struct fm10k_hw *hw)
  78 {
  79         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  80 }
  81
  82 static void
  83 fm10k_mbx_lock(struct fm10k_hw *hw)
  84 {
  85         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  86                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  87 }
  88
  89 static void
  90 fm10k_mbx_unlock(struct fm10k_hw *hw)
  91 {
  92         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  93 }
  94
  95 /*
  96  * reset queue to initial state, allocate software buffers used when starting
  97  * device.
  98  * return 0 on success
  99  * return -ENOMEM if buffers cannot be allocated
 100  * return -EINVAL if buffers do not satisfy alignment condition
 101  */
 102 static inline int
 103 rx_queue_reset(struct fm10k_rx_queue *q)
 104 {
 105         uint64_t dma_addr;
 106         int i, diag;
 107         PMD_INIT_FUNC_TRACE();
 108
 109         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 110         if (diag != 0)
 111                 return -ENOMEM;
 112
 113         for (i = 0; i < q->nb_desc; ++i) {
 114                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 115                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 116                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 117                                                 q->nb_desc);
 118                         return -EINVAL;
 119                 }
 120                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 121                 q->hw_ring[i].q.pkt_addr = dma_addr;
 122                 q->hw_ring[i].q.hdr_addr = dma_addr;
 123         }
 124
 125         q->next_dd = 0;
 126         q->next_alloc = 0;
 127         q->next_trigger = q->alloc_thresh - 1;
 128         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 129         q->rxrearm_start = 0;
 130         q->rxrearm_nb = 0;
 131
 132         return 0;
 133 }
 134
 135 /*
 136  * clean queue, descriptor rings, free software buffers used when stopping
 137  * device.
 138  */
 139 static inline void
 140 rx_queue_clean(struct fm10k_rx_queue *q)
 141 {
 142         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 143         uint32_t i;
 144         PMD_INIT_FUNC_TRACE();
 145
 146         /* zero descriptor rings */
 147         for (i = 0; i < q->nb_desc; ++i)
 148                 q->hw_ring[i] = zero;
 149
 150         /* vPMD driver has a different way of releasing mbufs. */
 151         if (q->rx_using_sse) {
 152                 fm10k_rx_queue_release_mbufs_vec(q);
 153                 return;
 154         }
 155
 156         /* free software buffers */
 157         for (i = 0; i < q->nb_desc; ++i) {
 158                 if (q->sw_ring[i]) {
 159                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 160                         q->sw_ring[i] = NULL;
 161                 }
 162         }
 163 }
 164
 165 /*
 166  * free all queue memory used when releasing the queue (i.e. configure)
 167  */
 168 static inline void
 169 rx_queue_free(struct fm10k_rx_queue *q)
 170 {
 171         PMD_INIT_FUNC_TRACE();
 172         if (q) {
 173                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 174                 rx_queue_clean(q);
 175                 if (q->sw_ring) {
 176                         rte_free(q->sw_ring);
 177                         q->sw_ring = NULL;
 178                 }
 179                 rte_free(q);
 180                 q = NULL;
 181         }
 182 }
 183
 184 /*
 185  * disable RX queue, wait unitl HW finished necessary flush operation
 186  */
 187 static inline int
 188 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 189 {
 190         uint32_t reg, i;
 191
 192         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 193         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 194                         reg & ~FM10K_RXQCTL_ENABLE);
 195
 196         /* Wait 100us at most */
 197         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 198                 rte_delay_us(1);
 199                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 200                 if (!(reg & FM10K_RXQCTL_ENABLE))
 201                         break;
 202         }
 203
 204         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 205                 return -1;
 206
 207         return 0;
 208 }
 209
 210 /*
 211  * reset queue to initial state, allocate software buffers used when starting
 212  * device
 213  */
 214 static inline void
 215 tx_queue_reset(struct fm10k_tx_queue *q)
 216 {
 217         PMD_INIT_FUNC_TRACE();
 218         q->last_free = 0;
 219         q->next_free = 0;
 220         q->nb_used = 0;
 221         q->nb_free = q->nb_desc - 1;
 222         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 223         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 224 }
 225
 226 /*
 227  * clean queue, descriptor rings, free software buffers used when stopping
 228  * device
 229  */
 230 static inline void
 231 tx_queue_clean(struct fm10k_tx_queue *q)
 232 {
 233         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 234         uint32_t i;
 235         PMD_INIT_FUNC_TRACE();
 236
 237         /* zero descriptor rings */
 238         for (i = 0; i < q->nb_desc; ++i)
 239                 q->hw_ring[i] = zero;
 240
 241         /* free software buffers */
 242         for (i = 0; i < q->nb_desc; ++i) {
 243                 if (q->sw_ring[i]) {
 244                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 245                         q->sw_ring[i] = NULL;
 246                 }
 247         }
 248 }
 249
 250 /*
 251  * free all queue memory used when releasing the queue (i.e. configure)
 252  */
 253 static inline void
 254 tx_queue_free(struct fm10k_tx_queue *q)
 255 {
 256         PMD_INIT_FUNC_TRACE();
 257         if (q) {
 258                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 259                 tx_queue_clean(q);
 260                 if (q->rs_tracker.list) {
 261                         rte_free(q->rs_tracker.list);
 262                         q->rs_tracker.list = NULL;
 263                 }
 264                 if (q->sw_ring) {
 265                         rte_free(q->sw_ring);
 266                         q->sw_ring = NULL;
 267                 }
 268                 rte_free(q);
 269                 q = NULL;
 270         }
 271 }
 272
 273 /*
 274  * disable TX queue, wait unitl HW finished necessary flush operation
 275  */
 276 static inline int
 277 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 278 {
 279         uint32_t reg, i;
 280
 281         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 282         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 283                         reg & ~FM10K_TXDCTL_ENABLE);
 284
 285         /* Wait 100us at most */
 286         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 287                 rte_delay_us(1);
 288                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 289                 if (!(reg & FM10K_TXDCTL_ENABLE))
 290                         break;
 291         }
 292
 293         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 294                 return -1;
 295
 296         return 0;
 297 }
 298
 299 static int
 300 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 301 {
 302         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 303         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 304         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 305         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 306
 307         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 308
 309         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 310                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 311                 return -EINVAL;
 312         }
 313
 314         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 315                 return 0;
 316
 317         if (hw->mac.type == fm10k_mac_vf) {
 318                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 319                 return -EINVAL;
 320         }
 321
 322         /* Check VMDQ queue pool number */
 323         if (vmdq_conf->nb_queue_pools >
 324                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 325                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 326                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 327                         vmdq_conf->nb_queue_pools);
 328                 return -EINVAL;
 329         }
 330
 331         return 0;
 332 }
 333
 334 static const struct fm10k_txq_ops def_txq_ops = {
 335         .release_mbufs = tx_queue_free,
 336         .reset = tx_queue_reset,
 337 };
 338
 339 static int
 340 fm10k_dev_configure(struct rte_eth_dev *dev)
 341 {
 342         int ret;
 343
 344         PMD_INIT_FUNC_TRACE();
 345
 346         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 347                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 348         /* multipe queue mode checking */
 349         ret  = fm10k_check_mq_mode(dev);
 350         if (ret != 0) {
 351                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 352                             ret);
 353                 return ret;
 354         }
 355
 356         return 0;
 357 }
 358
 359 /* fls = find last set bit = 32 minus the number of leading zeros */
 360 #ifndef fls
 361 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 362 #endif
 363
 364 static void
 365 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 366 {
 367         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 368         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 369         uint32_t i;
 370
 371         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 372
 373         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 374                 if (!vmdq_conf->pool_map[i].pools)
 375                         continue;
 376                 fm10k_mbx_lock(hw);
 377                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 378                 fm10k_mbx_unlock(hw);
 379         }
 380 }
 381
 382 static void
 383 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 384 {
 385         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 386
 387         /* Add default mac address */
 388         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 389                 MAIN_VSI_POOL_NUMBER);
 390 }
 391
 392 static void
 393 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 394 {
 395         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 396         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 397         uint32_t mrqc, *key, i, reta, j;
 398         uint64_t hf;
 399
 400 #define RSS_KEY_SIZE 40
 401         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 402                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 403                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 404                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 405                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 406                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 407         };
 408
 409         if (dev->data->nb_rx_queues == 1 ||
 410             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 411             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 412                 return;
 413
 414         /* random key is rss_intel_key (default) or user provided (rss_key) */
 415         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 416                 key = (uint32_t *)rss_intel_key;
 417         else
 418                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 419
 420         /* Now fill our hash function seeds, 4 bytes at a time */
 421         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 422                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 423
 424         /*
 425          * Fill in redirection table
 426          * The byte-swap is needed because NIC registers are in
 427          * little-endian order.
 428          */
 429         reta = 0;
 430         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 431                 if (j == dev->data->nb_rx_queues)
 432                         j = 0;
 433                 reta = (reta << CHAR_BIT) | j;
 434                 if ((i & 3) == 3)
 435                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 436                                         rte_bswap32(reta));
 437         }
 438
 439         /*
 440          * Generate RSS hash based on packet types, TCP/UDP
 441          * port numbers and/or IPv4/v6 src and dst addresses
 442          */
 443         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 444         mrqc = 0;
 445         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 446         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 447         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 448         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 449         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 450         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 451         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 452         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 453         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 454
 455         if (mrqc == 0) {
 456                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 457                         "supported", hf);
 458                 return;
 459         }
 460
 461         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 462 }
 463
 464 static void
 465 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 466         uint16_t nb_lport_old, uint16_t nb_lport_new)
 467 {
 468         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 469         uint32_t i;
 470
 471         fm10k_mbx_lock(hw);
 472         /* Disable previous logic ports */
 473         if (nb_lport_old)
 474                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 475                         nb_lport_old, false);
 476         /* Enable new logic ports */
 477         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 478                 nb_lport_new, true);
 479         fm10k_mbx_unlock(hw);
 480
 481         for (i = 0; i < nb_lport_new; i++) {
 482                 /* Set unicast mode by default. App can change
 483                  * to other mode in other API func.
 484                  */
 485                 fm10k_mbx_lock(hw);
 486                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 487                         FM10K_XCAST_MODE_NONE);
 488                 fm10k_mbx_unlock(hw);
 489         }
 490 }
 491
 492 static void
 493 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 494 {
 495         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 496         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 497         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 498         struct fm10k_macvlan_filter_info *macvlan;
 499         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 500         uint16_t nb_lport_new, nb_lport_old;
 501
 502         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 503         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 504
 505         fm10k_dev_rss_configure(dev);
 506
 507         /* only PF supports VMDQ */
 508         if (hw->mac.type != fm10k_mac_pf)
 509                 return;
 510
 511         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 512                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 513
 514         /* no pool number change, no need to update logic port and VLAN/MAC */
 515         if (macvlan->nb_queue_pools == nb_queue_pools)
 516                 return;
 517
 518         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 519         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 520         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 521
 522         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 523         memset(dev->data->mac_addrs, 0,
 524                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 525         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 526                 &dev->data->mac_addrs[0]);
 527         memset(macvlan, 0, sizeof(*macvlan));
 528         macvlan->nb_queue_pools = nb_queue_pools;
 529
 530         if (nb_queue_pools)
 531                 fm10k_dev_vmdq_rx_configure(dev);
 532         else
 533                 fm10k_dev_pf_main_vsi_reset(dev);
 534 }
 535
 536 static int
 537 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 538 {
 539         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 540         int i, ret;
 541         struct fm10k_tx_queue *txq;
 542         uint64_t base_addr;
 543         uint32_t size;
 544
 545         /* Disable TXINT to avoid possible interrupt */
 546         for (i = 0; i < hw->mac.max_queues; i++)
 547                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 548                                 3 << FM10K_TXINT_TIMER_SHIFT);
 549
 550         /* Setup TX queue */
 551         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 552                 txq = dev->data->tx_queues[i];
 553                 base_addr = txq->hw_ring_phys_addr;
 554                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 555
 556                 /* disable queue to avoid issues while updating state */
 557                 ret = tx_queue_disable(hw, i);
 558                 if (ret) {
 559                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 560                         return -1;
 561                 }
 562
 563                 /* set location and size for descriptor ring */
 564                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 565                                 base_addr & UINT64_LOWER_32BITS_MASK);
 566                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 567                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 568                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 569         }
 570
 571         /* set up vector or scalar TX function as appropriate */
 572         fm10k_set_tx_function(dev);
 573
 574         return 0;
 575 }
 576
 577 static int
 578 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 579 {
 580         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 581         int i, ret;
 582         struct fm10k_rx_queue *rxq;
 583         uint64_t base_addr;
 584         uint32_t size;
 585         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 586         uint16_t buf_size;
 587
 588         /* Disable RXINT to avoid possible interrupt */
 589         for (i = 0; i < hw->mac.max_queues; i++)
 590                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 591                                 3 << FM10K_RXINT_TIMER_SHIFT);
 592
 593         /* Setup RX queues */
 594         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 595                 rxq = dev->data->rx_queues[i];
 596                 base_addr = rxq->hw_ring_phys_addr;
 597                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 598
 599                 /* disable queue to avoid issues while updating state */
 600                 ret = rx_queue_disable(hw, i);
 601                 if (ret) {
 602                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 603                         return -1;
 604                 }
 605
 606                 /* Setup the Base and Length of the Rx Descriptor Ring */
 607                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 608                                 base_addr & UINT64_LOWER_32BITS_MASK);
 609                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 610                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 611                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 612
 613                 /* Configure the Rx buffer size for one buff without split */
 614                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 615                         RTE_PKTMBUF_HEADROOM);
 616                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 617                  * reserved for this purpose, and the worst case could be 511B.
 618                  * But SRR reg assumes all buffers have the same size. In order
 619                  * to fill the gap, we'll have to consider the worst case and
 620                  * assume 512B is reserved. If we don't do so, it's possible
 621                  * for HW to overwrite data to next mbuf.
 622                  */
 623                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 624
 625                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 626                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 627
 628                 /* It adds dual VLAN length for supporting dual VLAN */
 629                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 630                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 631                         dev->data->dev_conf.rxmode.enable_scatter) {
 632                         uint32_t reg;
 633                         dev->data->scattered_rx = 1;
 634                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 635                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 636                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 637                 }
 638
 639                 /* Enable drop on empty, it's RO for VF */
 640                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 641                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 642
 643                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 644                 FM10K_WRITE_FLUSH(hw);
 645         }
 646
 647         /* Configure VMDQ/RSS if applicable */
 648         fm10k_dev_mq_rx_configure(dev);
 649
 650         /* Decide the best RX function */
 651         fm10k_set_rx_function(dev);
 652
 653         return 0;
 654 }
 655
 656 static int
 657 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 658 {
 659         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 660         int err = -1;
 661         uint32_t reg;
 662         struct fm10k_rx_queue *rxq;
 663
 664         PMD_INIT_FUNC_TRACE();
 665
 666         if (rx_queue_id < dev->data->nb_rx_queues) {
 667                 rxq = dev->data->rx_queues[rx_queue_id];
 668                 err = rx_queue_reset(rxq);
 669                 if (err == -ENOMEM) {
 670                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 671                         return err;
 672                 } else if (err == -EINVAL) {
 673                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 674                                 " %d", err);
 675                         return err;
 676                 }
 677
 678                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 679                  * Note: this must be done AFTER the queue is enabled on real
 680                  * hardware, but BEFORE the queue is enabled when using the
 681                  * emulation platform. Do it in both places for now and remove
 682                  * this comment and the following two register writes when the
 683                  * emulation platform is no longer being used.
 684                  */
 685                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 686                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 687
 688                 /* Set PF ownership flag for PF devices */
 689                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 690                 if (hw->mac.type == fm10k_mac_pf)
 691                         reg |= FM10K_RXQCTL_PF;
 692                 reg |= FM10K_RXQCTL_ENABLE;
 693                 /* enable RX queue */
 694                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 695                 FM10K_WRITE_FLUSH(hw);
 696
 697                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 698                  * Note: this must be done AFTER the queue is enabled
 699                  */
 700                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 701                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 702         }
 703
 704         return err;
 705 }
 706
 707 static int
 708 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 709 {
 710         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 711
 712         PMD_INIT_FUNC_TRACE();
 713
 714         if (rx_queue_id < dev->data->nb_rx_queues) {
 715                 /* Disable RX queue */
 716                 rx_queue_disable(hw, rx_queue_id);
 717
 718                 /* Free mbuf and clean HW ring */
 719                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 720         }
 721
 722         return 0;
 723 }
 724
 725 static int
 726 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 727 {
 728         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 729         /** @todo - this should be defined in the shared code */
 730 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 731         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 732         int err = 0;
 733
 734         PMD_INIT_FUNC_TRACE();
 735
 736         if (tx_queue_id < dev->data->nb_tx_queues) {
 737                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 738
 739                 q->ops->reset(q);
 740
 741                 /* reset head and tail pointers */
 742                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 743                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 744
 745                 /* enable TX queue */
 746                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 747                                         FM10K_TXDCTL_ENABLE | txdctl);
 748                 FM10K_WRITE_FLUSH(hw);
 749         } else
 750                 err = -1;
 751
 752         return err;
 753 }
 754
 755 static int
 756 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 757 {
 758         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 759
 760         PMD_INIT_FUNC_TRACE();
 761
 762         if (tx_queue_id < dev->data->nb_tx_queues) {
 763                 tx_queue_disable(hw, tx_queue_id);
 764                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 765         }
 766
 767         return 0;
 768 }
 769
 770 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 771 {
 772         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 773                 != FM10K_DGLORTMAP_NONE);
 774 }
 775
 776 static void
 777 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 778 {
 779         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 780         int status;
 781
 782         PMD_INIT_FUNC_TRACE();
 783
 784         /* Return if it didn't acquire valid glort range */
 785         if (!fm10k_glort_valid(hw))
 786                 return;
 787
 788         fm10k_mbx_lock(hw);
 789         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 790                                 FM10K_XCAST_MODE_PROMISC);
 791         fm10k_mbx_unlock(hw);
 792
 793         if (status != FM10K_SUCCESS)
 794                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 795 }
 796
 797 static void
 798 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 799 {
 800         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 801         uint8_t mode;
 802         int status;
 803
 804         PMD_INIT_FUNC_TRACE();
 805
 806         /* Return if it didn't acquire valid glort range */
 807         if (!fm10k_glort_valid(hw))
 808                 return;
 809
 810         if (dev->data->all_multicast == 1)
 811                 mode = FM10K_XCAST_MODE_ALLMULTI;
 812         else
 813                 mode = FM10K_XCAST_MODE_NONE;
 814
 815         fm10k_mbx_lock(hw);
 816         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 817                                 mode);
 818         fm10k_mbx_unlock(hw);
 819
 820         if (status != FM10K_SUCCESS)
 821                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 822 }
 823
 824 static void
 825 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 826 {
 827         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 828         int status;
 829
 830         PMD_INIT_FUNC_TRACE();
 831
 832         /* Return if it didn't acquire valid glort range */
 833         if (!fm10k_glort_valid(hw))
 834                 return;
 835
 836         /* If promiscuous mode is enabled, it doesn't make sense to enable
 837          * allmulticast and disable promiscuous since fm10k only can select
 838          * one of the modes.
 839          */
 840         if (dev->data->promiscuous) {
 841                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 842                         "needn't enable allmulticast");
 843                 return;
 844         }
 845
 846         fm10k_mbx_lock(hw);
 847         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 848                                 FM10K_XCAST_MODE_ALLMULTI);
 849         fm10k_mbx_unlock(hw);
 850
 851         if (status != FM10K_SUCCESS)
 852                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 853 }
 854
 855 static void
 856 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 857 {
 858         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859         int status;
 860
 861         PMD_INIT_FUNC_TRACE();
 862
 863         /* Return if it didn't acquire valid glort range */
 864         if (!fm10k_glort_valid(hw))
 865                 return;
 866
 867         if (dev->data->promiscuous) {
 868                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 869                         "since promisc mode is enabled");
 870                 return;
 871         }
 872
 873         fm10k_mbx_lock(hw);
 874         /* Change mode to unicast mode */
 875         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 876                                 FM10K_XCAST_MODE_NONE);
 877         fm10k_mbx_unlock(hw);
 878
 879         if (status != FM10K_SUCCESS)
 880                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 881 }
 882
 883 static void
 884 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 885 {
 886         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 887         uint32_t dglortdec, pool_len, rss_len, i;
 888         uint16_t nb_queue_pools;
 889         struct fm10k_macvlan_filter_info *macvlan;
 890
 891         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 892         nb_queue_pools = macvlan->nb_queue_pools;
 893         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 894         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 895         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 896
 897         /* Establish only MAP 0 as valid */
 898         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 899
 900         /* Configure VMDQ/RSS DGlort Decoder */
 901         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 902
 903         /* Invalidate all other GLORT entries */
 904         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 905                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 906                                 FM10K_DGLORTMAP_NONE);
 907 }
 908
 909 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 910 static int
 911 fm10k_dev_start(struct rte_eth_dev *dev)
 912 {
 913         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 914         int i, diag;
 915
 916         PMD_INIT_FUNC_TRACE();
 917
 918         /* stop, init, then start the hw */
 919         diag = fm10k_stop_hw(hw);
 920         if (diag != FM10K_SUCCESS) {
 921                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 922                 return -EIO;
 923         }
 924
 925         diag = fm10k_init_hw(hw);
 926         if (diag != FM10K_SUCCESS) {
 927                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 928                 return -EIO;
 929         }
 930
 931         diag = fm10k_start_hw(hw);
 932         if (diag != FM10K_SUCCESS) {
 933                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 934                 return -EIO;
 935         }
 936
 937         diag = fm10k_dev_tx_init(dev);
 938         if (diag) {
 939                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 940                 return diag;
 941         }
 942
 943         diag = fm10k_dev_rx_init(dev);
 944         if (diag) {
 945                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 946                 return diag;
 947         }
 948
 949         if (hw->mac.type == fm10k_mac_pf)
 950                 fm10k_dev_dglort_map_configure(dev);
 951
 952         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 953                 struct fm10k_rx_queue *rxq;
 954                 rxq = dev->data->rx_queues[i];
 955
 956                 if (rxq->rx_deferred_start)
 957                         continue;
 958                 diag = fm10k_dev_rx_queue_start(dev, i);
 959                 if (diag != 0) {
 960                         int j;
 961                         for (j = 0; j < i; ++j)
 962                                 rx_queue_clean(dev->data->rx_queues[j]);
 963                         return diag;
 964                 }
 965         }
 966
 967         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 968                 struct fm10k_tx_queue *txq;
 969                 txq = dev->data->tx_queues[i];
 970
 971                 if (txq->tx_deferred_start)
 972                         continue;
 973                 diag = fm10k_dev_tx_queue_start(dev, i);
 974                 if (diag != 0) {
 975                         int j;
 976                         for (j = 0; j < i; ++j)
 977                                 tx_queue_clean(dev->data->tx_queues[j]);
 978                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 979                                 rx_queue_clean(dev->data->rx_queues[j]);
 980                         return diag;
 981                 }
 982         }
 983
 984         /* Update default vlan when not in VMDQ mode */
 985         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 986                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 987
 988         return 0;
 989 }
 990
 991 static void
 992 fm10k_dev_stop(struct rte_eth_dev *dev)
 993 {
 994         int i;
 995
 996         PMD_INIT_FUNC_TRACE();
 997
 998         if (dev->data->tx_queues)
 999                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1000                         fm10k_dev_tx_queue_stop(dev, i);
1001
1002         if (dev->data->rx_queues)
1003                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1004                         fm10k_dev_rx_queue_stop(dev, i);
1005 }
1006
1007 static void
1008 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1009 {
1010         int i;
1011
1012         PMD_INIT_FUNC_TRACE();
1013
1014         if (dev->data->tx_queues) {
1015                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1016                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1017
1018                         txq->ops->release_mbufs(txq);
1019                 }
1020         }
1021
1022         if (dev->data->rx_queues) {
1023                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1024                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1025         }
1026 }
1027
1028 static void
1029 fm10k_dev_close(struct rte_eth_dev *dev)
1030 {
1031         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1032         uint16_t nb_lport;
1033         struct fm10k_macvlan_filter_info *macvlan;
1034
1035         PMD_INIT_FUNC_TRACE();
1036
1037         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1038         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1039         fm10k_mbx_lock(hw);
1040         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1041                 nb_lport, false);
1042         fm10k_mbx_unlock(hw);
1043
1044         /* Stop mailbox service first */
1045         fm10k_close_mbx_service(hw);
1046         fm10k_dev_stop(dev);
1047         fm10k_dev_queue_release(dev);
1048         fm10k_stop_hw(hw);
1049 }
1050
1051 static int
1052 fm10k_link_update(struct rte_eth_dev *dev,
1053         __rte_unused int wait_to_complete)
1054 {
1055         PMD_INIT_FUNC_TRACE();
1056
1057         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1058          * x8 PCIe interface. For now, we leave the speed undefined since there
1059          * is no 50Gbps Ethernet. */
1060         dev->data->dev_link.link_speed  = 0;
1061         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1062         dev->data->dev_link.link_status = 1;
1063
1064         return 0;
1065 }
1066
1067 static void
1068 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1069 {
1070         uint64_t ipackets, opackets, ibytes, obytes;
1071         struct fm10k_hw *hw =
1072                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1073         struct fm10k_hw_stats *hw_stats =
1074                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1075         int i;
1076
1077         PMD_INIT_FUNC_TRACE();
1078
1079         fm10k_update_hw_stats(hw, hw_stats);
1080
1081         ipackets = opackets = ibytes = obytes = 0;
1082         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1083                 (i < hw->mac.max_queues); ++i) {
1084                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1085                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1086                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1087                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1088                 ipackets += stats->q_ipackets[i];
1089                 opackets += stats->q_opackets[i];
1090                 ibytes   += stats->q_ibytes[i];
1091                 obytes   += stats->q_obytes[i];
1092         }
1093         stats->ipackets = ipackets;
1094         stats->opackets = opackets;
1095         stats->ibytes = ibytes;
1096         stats->obytes = obytes;
1097 }
1098
1099 static void
1100 fm10k_stats_reset(struct rte_eth_dev *dev)
1101 {
1102         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1103         struct fm10k_hw_stats *hw_stats =
1104                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1105
1106         PMD_INIT_FUNC_TRACE();
1107
1108         memset(hw_stats, 0, sizeof(*hw_stats));
1109         fm10k_rebind_hw_stats(hw, hw_stats);
1110 }
1111
1112 static void
1113 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1114         struct rte_eth_dev_info *dev_info)
1115 {
1116         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1117
1118         PMD_INIT_FUNC_TRACE();
1119
1120         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1121         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1122         dev_info->max_rx_queues      = hw->mac.max_queues;
1123         dev_info->max_tx_queues      = hw->mac.max_queues;
1124         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1125         dev_info->max_hash_mac_addrs = 0;
1126         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1127         dev_info->vmdq_pool_base     = 0;
1128         dev_info->vmdq_queue_base    = 0;
1129         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1130         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1131         dev_info->rx_offload_capa =
1132                 DEV_RX_OFFLOAD_VLAN_STRIP |
1133                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1134                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1135                 DEV_RX_OFFLOAD_TCP_CKSUM;
1136         dev_info->tx_offload_capa =
1137                 DEV_TX_OFFLOAD_VLAN_INSERT |
1138                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1139                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1140                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1141                 DEV_TX_OFFLOAD_TCP_TSO;
1142
1143         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1144         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1145
1146         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1147                 .rx_thresh = {
1148                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1149                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1150                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1151                 },
1152                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1153                 .rx_drop_en = 0,
1154         };
1155
1156         dev_info->default_txconf = (struct rte_eth_txconf) {
1157                 .tx_thresh = {
1158                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1159                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1160                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1161                 },
1162                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1163                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1164                 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1165         };
1166
1167         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1168                 .nb_max = FM10K_MAX_RX_DESC,
1169                 .nb_min = FM10K_MIN_RX_DESC,
1170                 .nb_align = FM10K_MULT_RX_DESC,
1171         };
1172
1173         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1174                 .nb_max = FM10K_MAX_TX_DESC,
1175                 .nb_min = FM10K_MIN_TX_DESC,
1176                 .nb_align = FM10K_MULT_TX_DESC,
1177         };
1178 }
1179
1180 static int
1181 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1182 {
1183         s32 result;
1184         uint16_t mac_num = 0;
1185         uint32_t vid_idx, vid_bit, mac_index;
1186         struct fm10k_hw *hw;
1187         struct fm10k_macvlan_filter_info *macvlan;
1188         struct rte_eth_dev_data *data = dev->data;
1189
1190         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1191         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1192
1193         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1194                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1195                 return (-EINVAL);
1196         }
1197
1198         if (vlan_id > ETH_VLAN_ID_MAX) {
1199                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1200                 return (-EINVAL);
1201         }
1202
1203         vid_idx = FM10K_VFTA_IDX(vlan_id);
1204         vid_bit = FM10K_VFTA_BIT(vlan_id);
1205         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1206         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1207                 return 0;
1208         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1209         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1210                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1211                         "in the VLAN filter table");
1212                 return (-EINVAL);
1213         }
1214
1215         fm10k_mbx_lock(hw);
1216         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1217         fm10k_mbx_unlock(hw);
1218         if (result != FM10K_SUCCESS) {
1219                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1220                 return (-EIO);
1221         }
1222
1223         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1224                         (result == FM10K_SUCCESS); mac_index++) {
1225                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1226                         continue;
1227                 if (mac_num > macvlan->mac_num - 1) {
1228                         PMD_INIT_LOG(ERR, "MAC address number "
1229                                         "not match");
1230                         break;
1231                 }
1232                 fm10k_mbx_lock(hw);
1233                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1234                         data->mac_addrs[mac_index].addr_bytes,
1235                         vlan_id, on, 0);
1236                 fm10k_mbx_unlock(hw);
1237                 mac_num++;
1238         }
1239         if (result != FM10K_SUCCESS) {
1240                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1241                 return (-EIO);
1242         }
1243
1244         if (on) {
1245                 macvlan->vlan_num++;
1246                 macvlan->vfta[vid_idx] |= vid_bit;
1247         } else {
1248                 macvlan->vlan_num--;
1249                 macvlan->vfta[vid_idx] &= ~vid_bit;
1250         }
1251         return 0;
1252 }
1253
1254 static void
1255 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1256 {
1257         if (mask & ETH_VLAN_STRIP_MASK) {
1258                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1259                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1260                                         "always on in fm10k");
1261         }
1262
1263         if (mask & ETH_VLAN_EXTEND_MASK) {
1264                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1265                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1266                                         "supported in fm10k");
1267         }
1268
1269         if (mask & ETH_VLAN_FILTER_MASK) {
1270                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1271                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1272         }
1273 }
1274
1275 /* Add/Remove a MAC address, and update filters to main VSI */
1276 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1277                 const u8 *mac, bool add, uint32_t pool)
1278 {
1279         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1280         struct fm10k_macvlan_filter_info *macvlan;
1281         uint32_t i, j, k;
1282
1283         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1284
1285         if (pool != MAIN_VSI_POOL_NUMBER) {
1286                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1287                         "mac to pool %u", pool);
1288                 return;
1289         }
1290         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1291                 if (!macvlan->vfta[j])
1292                         continue;
1293                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1294                         if (!(macvlan->vfta[j] & (1 << k)))
1295                                 continue;
1296                         if (i + 1 > macvlan->vlan_num) {
1297                                 PMD_INIT_LOG(ERR, "vlan number not match");
1298                                 return;
1299                         }
1300                         fm10k_mbx_lock(hw);
1301                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1302                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1303                         fm10k_mbx_unlock(hw);
1304                         i++;
1305                 }
1306         }
1307 }
1308
1309 /* Add/Remove a MAC address, and update filters to VMDQ */
1310 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1311                 const u8 *mac, bool add, uint32_t pool)
1312 {
1313         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1314         struct fm10k_macvlan_filter_info *macvlan;
1315         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1316         uint32_t i;
1317
1318         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1319         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1320
1321         if (pool > macvlan->nb_queue_pools) {
1322                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1323                         " Max pool is %u",
1324                         pool, macvlan->nb_queue_pools);
1325                 return;
1326         }
1327         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1328                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1329                         continue;
1330                 fm10k_mbx_lock(hw);
1331                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1332                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1333                 fm10k_mbx_unlock(hw);
1334         }
1335 }
1336
1337 /* Add/Remove a MAC address, and update filters */
1338 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1339                 const u8 *mac, bool add, uint32_t pool)
1340 {
1341         struct fm10k_macvlan_filter_info *macvlan;
1342
1343         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1344
1345         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1346                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1347         else
1348                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1349
1350         if (add)
1351                 macvlan->mac_num++;
1352         else
1353                 macvlan->mac_num--;
1354 }
1355
1356 /* Add a MAC address, and update filters */
1357 static void
1358 fm10k_macaddr_add(struct rte_eth_dev *dev,
1359                 struct ether_addr *mac_addr,
1360                 uint32_t index,
1361                 uint32_t pool)
1362 {
1363         struct fm10k_macvlan_filter_info *macvlan;
1364
1365         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1366         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1367         macvlan->mac_vmdq_id[index] = pool;
1368 }
1369
1370 /* Remove a MAC address, and update filters */
1371 static void
1372 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1373 {
1374         struct rte_eth_dev_data *data = dev->data;
1375         struct fm10k_macvlan_filter_info *macvlan;
1376
1377         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1378         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1379                         FALSE, macvlan->mac_vmdq_id[index]);
1380         macvlan->mac_vmdq_id[index] = 0;
1381 }
1382
1383 static inline int
1384 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1385 {
1386         if ((request < min) || (request > max) || ((request % mult) != 0))
1387                 return -1;
1388         else
1389                 return 0;
1390 }
1391
1392 /*
1393  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1394  * the physical address is required. If the memzone is already created, then
1395  * this function returns a pointer to the existing memzone.
1396  */
1397 static inline const struct rte_memzone *
1398 allocate_hw_ring(const char *driver_name, const char *ring_name,
1399         uint8_t port_id, uint16_t queue_id, int socket_id,
1400         uint32_t size, uint32_t align)
1401 {
1402         char name[RTE_MEMZONE_NAMESIZE];
1403         const struct rte_memzone *mz;
1404
1405         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1406                  driver_name, ring_name, port_id, queue_id, socket_id);
1407
1408         /* return the memzone if it already exists */
1409         mz = rte_memzone_lookup(name);
1410         if (mz)
1411                 return mz;
1412
1413 #ifdef RTE_LIBRTE_XEN_DOM0
1414         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1415                                            RTE_PGSIZE_2M);
1416 #else
1417         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1418 #endif
1419 }
1420
1421 static inline int
1422 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1423 {
1424         if ((request < min) || (request > max) || ((div % request) != 0))
1425                 return -1;
1426         else
1427                 return 0;
1428 }
1429
1430 static inline int
1431 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1432 {
1433         uint16_t rx_free_thresh;
1434
1435         if (conf->rx_free_thresh == 0)
1436                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1437         else
1438                 rx_free_thresh = conf->rx_free_thresh;
1439
1440         /* make sure the requested threshold satisfies the constraints */
1441         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1442                         FM10K_RX_FREE_THRESH_MAX(q),
1443                         FM10K_RX_FREE_THRESH_DIV(q),
1444                         rx_free_thresh)) {
1445                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1446                         "less than or equal to %u, "
1447                         "greater than or equal to %u, "
1448                         "and a divisor of %u",
1449                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1450                         FM10K_RX_FREE_THRESH_MIN(q),
1451                         FM10K_RX_FREE_THRESH_DIV(q));
1452                 return (-EINVAL);
1453         }
1454
1455         q->alloc_thresh = rx_free_thresh;
1456         q->drop_en = conf->rx_drop_en;
1457         q->rx_deferred_start = conf->rx_deferred_start;
1458
1459         return 0;
1460 }
1461
1462 /*
1463  * Hardware requires specific alignment for Rx packet buffers. At
1464  * least one of the following two conditions must be satisfied.
1465  *  1. Address is 512B aligned
1466  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1467  *
1468  * As such, the driver may need to adjust the DMA address within the
1469  * buffer by up to 512B.
1470  *
1471  * return 1 if the element size is valid, otherwise return 0.
1472  */
1473 static int
1474 mempool_element_size_valid(struct rte_mempool *mp)
1475 {
1476         uint32_t min_size;
1477
1478         /* elt_size includes mbuf header and headroom */
1479         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1480                         RTE_PKTMBUF_HEADROOM;
1481
1482         /* account for up to 512B of alignment */
1483         min_size -= FM10K_RX_DATABUF_ALIGN;
1484
1485         /* sanity check for overflow */
1486         if (min_size > mp->elt_size)
1487                 return 0;
1488
1489         /* size is valid */
1490         return 1;
1491 }
1492
1493 static int
1494 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1495         uint16_t nb_desc, unsigned int socket_id,
1496         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1497 {
1498         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1499         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1500         struct fm10k_rx_queue *q;
1501         const struct rte_memzone *mz;
1502
1503         PMD_INIT_FUNC_TRACE();
1504
1505         /* make sure the mempool element size can account for alignment. */
1506         if (!mempool_element_size_valid(mp)) {
1507                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1508                 return (-EINVAL);
1509         }
1510
1511         /* make sure a valid number of descriptors have been requested */
1512         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1513                                 FM10K_MULT_RX_DESC, nb_desc)) {
1514                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1515                         "less than or equal to %"PRIu32", "
1516                         "greater than or equal to %u, "
1517                         "and a multiple of %u",
1518                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1519                         FM10K_MULT_RX_DESC);
1520                 return (-EINVAL);
1521         }
1522
1523         /*
1524          * if this queue existed already, free the associated memory. The
1525          * queue cannot be reused in case we need to allocate memory on
1526          * different socket than was previously used.
1527          */
1528         if (dev->data->rx_queues[queue_id] != NULL) {
1529                 rx_queue_free(dev->data->rx_queues[queue_id]);
1530                 dev->data->rx_queues[queue_id] = NULL;
1531         }
1532
1533         /* allocate memory for the queue structure */
1534         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1535                                 socket_id);
1536         if (q == NULL) {
1537                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1538                 return (-ENOMEM);
1539         }
1540
1541         /* setup queue */
1542         q->mp = mp;
1543         q->nb_desc = nb_desc;
1544         q->port_id = dev->data->port_id;
1545         q->queue_id = queue_id;
1546         q->tail_ptr = (volatile uint32_t *)
1547                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1548         if (handle_rxconf(q, conf))
1549                 return (-EINVAL);
1550
1551         /* allocate memory for the software ring */
1552         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1553                                         nb_desc * sizeof(struct rte_mbuf *),
1554                                         RTE_CACHE_LINE_SIZE, socket_id);
1555         if (q->sw_ring == NULL) {
1556                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1557                 rte_free(q);
1558                 return (-ENOMEM);
1559         }
1560
1561         /*
1562          * allocate memory for the hardware descriptor ring. A memzone large
1563          * enough to hold the maximum ring size is requested to allow for
1564          * resizing in later calls to the queue setup function.
1565          */
1566         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1567                                 dev->data->port_id, queue_id, socket_id,
1568                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1569         if (mz == NULL) {
1570                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1571                 rte_free(q->sw_ring);
1572                 rte_free(q);
1573                 return (-ENOMEM);
1574         }
1575         q->hw_ring = mz->addr;
1576 #ifdef RTE_LIBRTE_XEN_DOM0
1577         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1578 #else
1579         q->hw_ring_phys_addr = mz->phys_addr;
1580 #endif
1581
1582         /* Check if number of descs satisfied Vector requirement */
1583         if (!rte_is_power_of_2(nb_desc)) {
1584                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1585                                     "preconditions - canceling the feature for "
1586                                     "the whole port[%d]",
1587                              q->queue_id, q->port_id);
1588                 dev_info->rx_vec_allowed = false;
1589         } else
1590                 fm10k_rxq_vec_setup(q);
1591
1592         dev->data->rx_queues[queue_id] = q;
1593         return 0;
1594 }
1595
1596 static void
1597 fm10k_rx_queue_release(void *queue)
1598 {
1599         PMD_INIT_FUNC_TRACE();
1600
1601         rx_queue_free(queue);
1602 }
1603
1604 static inline int
1605 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1606 {
1607         uint16_t tx_free_thresh;
1608         uint16_t tx_rs_thresh;
1609
1610         /* constraint MACROs require that tx_free_thresh is configured
1611          * before tx_rs_thresh */
1612         if (conf->tx_free_thresh == 0)
1613                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1614         else
1615                 tx_free_thresh = conf->tx_free_thresh;
1616
1617         /* make sure the requested threshold satisfies the constraints */
1618         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1619                         FM10K_TX_FREE_THRESH_MAX(q),
1620                         FM10K_TX_FREE_THRESH_DIV(q),
1621                         tx_free_thresh)) {
1622                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1623                         "less than or equal to %u, "
1624                         "greater than or equal to %u, "
1625                         "and a divisor of %u",
1626                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1627                         FM10K_TX_FREE_THRESH_MIN(q),
1628                         FM10K_TX_FREE_THRESH_DIV(q));
1629                 return (-EINVAL);
1630         }
1631
1632         q->free_thresh = tx_free_thresh;
1633
1634         if (conf->tx_rs_thresh == 0)
1635                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1636         else
1637                 tx_rs_thresh = conf->tx_rs_thresh;
1638
1639         q->tx_deferred_start = conf->tx_deferred_start;
1640
1641         /* make sure the requested threshold satisfies the constraints */
1642         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1643                         FM10K_TX_RS_THRESH_MAX(q),
1644                         FM10K_TX_RS_THRESH_DIV(q),
1645                         tx_rs_thresh)) {
1646                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1647                         "less than or equal to %u, "
1648                         "greater than or equal to %u, "
1649                         "and a divisor of %u",
1650                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1651                         FM10K_TX_RS_THRESH_MIN(q),
1652                         FM10K_TX_RS_THRESH_DIV(q));
1653                 return (-EINVAL);
1654         }
1655
1656         q->rs_thresh = tx_rs_thresh;
1657
1658         return 0;
1659 }
1660
1661 static int
1662 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1663         uint16_t nb_desc, unsigned int socket_id,
1664         const struct rte_eth_txconf *conf)
1665 {
1666         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1667         struct fm10k_tx_queue *q;
1668         const struct rte_memzone *mz;
1669
1670         PMD_INIT_FUNC_TRACE();
1671
1672         /* make sure a valid number of descriptors have been requested */
1673         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1674                                 FM10K_MULT_TX_DESC, nb_desc)) {
1675                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1676                         "less than or equal to %"PRIu32", "
1677                         "greater than or equal to %u, "
1678                         "and a multiple of %u",
1679                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1680                         FM10K_MULT_TX_DESC);
1681                 return (-EINVAL);
1682         }
1683
1684         /*
1685          * if this queue existed already, free the associated memory. The
1686          * queue cannot be reused in case we need to allocate memory on
1687          * different socket than was previously used.
1688          */
1689         if (dev->data->tx_queues[queue_id] != NULL) {
1690                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1691
1692                 txq->ops->release_mbufs(txq);
1693                 dev->data->tx_queues[queue_id] = NULL;
1694         }
1695
1696         /* allocate memory for the queue structure */
1697         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1698                                 socket_id);
1699         if (q == NULL) {
1700                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1701                 return (-ENOMEM);
1702         }
1703
1704         /* setup queue */
1705         q->nb_desc = nb_desc;
1706         q->port_id = dev->data->port_id;
1707         q->queue_id = queue_id;
1708         q->txq_flags = conf->txq_flags;
1709         q->ops = &def_txq_ops;
1710         q->tail_ptr = (volatile uint32_t *)
1711                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1712         if (handle_txconf(q, conf))
1713                 return (-EINVAL);
1714
1715         /* allocate memory for the software ring */
1716         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1717                                         nb_desc * sizeof(struct rte_mbuf *),
1718                                         RTE_CACHE_LINE_SIZE, socket_id);
1719         if (q->sw_ring == NULL) {
1720                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1721                 rte_free(q);
1722                 return (-ENOMEM);
1723         }
1724
1725         /*
1726          * allocate memory for the hardware descriptor ring. A memzone large
1727          * enough to hold the maximum ring size is requested to allow for
1728          * resizing in later calls to the queue setup function.
1729          */
1730         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1731                                 dev->data->port_id, queue_id, socket_id,
1732                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1733         if (mz == NULL) {
1734                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1735                 rte_free(q->sw_ring);
1736                 rte_free(q);
1737                 return (-ENOMEM);
1738         }
1739         q->hw_ring = mz->addr;
1740 #ifdef RTE_LIBRTE_XEN_DOM0
1741         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1742 #else
1743         q->hw_ring_phys_addr = mz->phys_addr;
1744 #endif
1745
1746         /*
1747          * allocate memory for the RS bit tracker. Enough slots to hold the
1748          * descriptor index for each RS bit needing to be set are required.
1749          */
1750         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1751                                 ((nb_desc + 1) / q->rs_thresh) *
1752                                 sizeof(uint16_t),
1753                                 RTE_CACHE_LINE_SIZE, socket_id);
1754         if (q->rs_tracker.list == NULL) {
1755                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1756                 rte_free(q->sw_ring);
1757                 rte_free(q);
1758                 return (-ENOMEM);
1759         }
1760
1761         dev->data->tx_queues[queue_id] = q;
1762         return 0;
1763 }
1764
1765 static void
1766 fm10k_tx_queue_release(void *queue)
1767 {
1768         struct fm10k_tx_queue *q = queue;
1769         PMD_INIT_FUNC_TRACE();
1770
1771         q->ops->release_mbufs(q);
1772 }
1773
1774 static int
1775 fm10k_reta_update(struct rte_eth_dev *dev,
1776                         struct rte_eth_rss_reta_entry64 *reta_conf,
1777                         uint16_t reta_size)
1778 {
1779         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1780         uint16_t i, j, idx, shift;
1781         uint8_t mask;
1782         uint32_t reta;
1783
1784         PMD_INIT_FUNC_TRACE();
1785
1786         if (reta_size > FM10K_MAX_RSS_INDICES) {
1787                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1788                         "(%d) doesn't match the number hardware can supported "
1789                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1790                 return -EINVAL;
1791         }
1792
1793         /*
1794          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1795          * 128-entries in 32 registers
1796          */
1797         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1798                 idx = i / RTE_RETA_GROUP_SIZE;
1799                 shift = i % RTE_RETA_GROUP_SIZE;
1800                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1801                                 BIT_MASK_PER_UINT32);
1802                 if (mask == 0)
1803                         continue;
1804
1805                 reta = 0;
1806                 if (mask != BIT_MASK_PER_UINT32)
1807                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1808
1809                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1810                         if (mask & (0x1 << j)) {
1811                                 if (mask != 0xF)
1812                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1813                                 reta |= reta_conf[idx].reta[shift + j] <<
1814                                                 (CHAR_BIT * j);
1815                         }
1816                 }
1817                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1818         }
1819
1820         return 0;
1821 }
1822
1823 static int
1824 fm10k_reta_query(struct rte_eth_dev *dev,
1825                         struct rte_eth_rss_reta_entry64 *reta_conf,
1826                         uint16_t reta_size)
1827 {
1828         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1829         uint16_t i, j, idx, shift;
1830         uint8_t mask;
1831         uint32_t reta;
1832
1833         PMD_INIT_FUNC_TRACE();
1834
1835         if (reta_size < FM10K_MAX_RSS_INDICES) {
1836                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1837                         "(%d) doesn't match the number hardware can supported "
1838                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1839                 return -EINVAL;
1840         }
1841
1842         /*
1843          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1844          * 128-entries in 32 registers
1845          */
1846         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1847                 idx = i / RTE_RETA_GROUP_SIZE;
1848                 shift = i % RTE_RETA_GROUP_SIZE;
1849                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1850                                 BIT_MASK_PER_UINT32);
1851                 if (mask == 0)
1852                         continue;
1853
1854                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1855                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1856                         if (mask & (0x1 << j))
1857                                 reta_conf[idx].reta[shift + j] = ((reta >>
1858                                         CHAR_BIT * j) & UINT8_MAX);
1859                 }
1860         }
1861
1862         return 0;
1863 }
1864
1865 static int
1866 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1867         struct rte_eth_rss_conf *rss_conf)
1868 {
1869         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1870         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1871         uint32_t mrqc;
1872         uint64_t hf = rss_conf->rss_hf;
1873         int i;
1874
1875         PMD_INIT_FUNC_TRACE();
1876
1877         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1878                 FM10K_RSSRK_ENTRIES_PER_REG)
1879                 return -EINVAL;
1880
1881         if (hf == 0)
1882                 return -EINVAL;
1883
1884         mrqc = 0;
1885         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1886         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1887         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1888         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1889         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1890         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1891         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1892         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1893         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1894
1895         /* If the mapping doesn't fit any supported, return */
1896         if (mrqc == 0)
1897                 return -EINVAL;
1898
1899         if (key != NULL)
1900                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1901                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1902
1903         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1904
1905         return 0;
1906 }
1907
1908 static int
1909 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1910         struct rte_eth_rss_conf *rss_conf)
1911 {
1912         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1913         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1914         uint32_t mrqc;
1915         uint64_t hf;
1916         int i;
1917
1918         PMD_INIT_FUNC_TRACE();
1919
1920         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1921                                 FM10K_RSSRK_ENTRIES_PER_REG)
1922                 return -EINVAL;
1923
1924         if (key != NULL)
1925                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1926                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1927
1928         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1929         hf = 0;
1930         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1931         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1932         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1933         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1934         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1935         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1936         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1937         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1938         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1939
1940         rss_conf->rss_hf = hf;
1941
1942         return 0;
1943 }
1944
1945 static void
1946 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1947 {
1948         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1949         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1950
1951         /* Bind all local non-queue interrupt to vector 0 */
1952         int_map |= 0;
1953
1954         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1955         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1956         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1957         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1958         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1959         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1960
1961         /* Enable misc causes */
1962         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1963                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1964                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1965                                 FM10K_EIMR_ENABLE(MAILBOX) |
1966                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1967                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1968                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1969                                 FM10K_EIMR_ENABLE(VFLR));
1970
1971         /* Enable ITR 0 */
1972         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1973                                         FM10K_ITR_MASK_CLEAR);
1974         FM10K_WRITE_FLUSH(hw);
1975 }
1976
1977 static void
1978 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1979 {
1980         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1981         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1982
1983         int_map |= 0;
1984
1985         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1986         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1987         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1988         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1989         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1990         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1991
1992         /* Disable misc causes */
1993         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1994                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1995                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1996                                 FM10K_EIMR_DISABLE(MAILBOX) |
1997                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1998                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1999                                 FM10K_EIMR_DISABLE(SRAMERROR) |
2000                                 FM10K_EIMR_DISABLE(VFLR));
2001
2002         /* Disable ITR 0 */
2003         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2004         FM10K_WRITE_FLUSH(hw);
2005 }
2006
2007 static void
2008 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2009 {
2010         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2011         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2012
2013         /* Bind all local non-queue interrupt to vector 0 */
2014         int_map |= 0;
2015
2016         /* Only INT 0 available, other 15 are reserved. */
2017         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2018
2019         /* Enable ITR 0 */
2020         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2021                                         FM10K_ITR_MASK_CLEAR);
2022         FM10K_WRITE_FLUSH(hw);
2023 }
2024
2025 static void
2026 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2027 {
2028         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2029         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2030
2031         int_map |= 0;
2032
2033         /* Only INT 0 available, other 15 are reserved. */
2034         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2035
2036         /* Disable ITR 0 */
2037         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2038         FM10K_WRITE_FLUSH(hw);
2039 }
2040
2041 static int
2042 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2043 {
2044         struct fm10k_fault fault;
2045         int err;
2046         const char *estr = "Unknown error";
2047
2048         /* Process PCA fault */
2049         if (eicr & FM10K_EICR_PCA_FAULT) {
2050                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2051                 if (err)
2052                         goto error;
2053                 switch (fault.type) {
2054                 case PCA_NO_FAULT:
2055                         estr = "PCA_NO_FAULT"; break;
2056                 case PCA_UNMAPPED_ADDR:
2057                         estr = "PCA_UNMAPPED_ADDR"; break;
2058                 case PCA_BAD_QACCESS_PF:
2059                         estr = "PCA_BAD_QACCESS_PF"; break;
2060                 case PCA_BAD_QACCESS_VF:
2061                         estr = "PCA_BAD_QACCESS_VF"; break;
2062                 case PCA_MALICIOUS_REQ:
2063                         estr = "PCA_MALICIOUS_REQ"; break;
2064                 case PCA_POISONED_TLP:
2065                         estr = "PCA_POISONED_TLP"; break;
2066                 case PCA_TLP_ABORT:
2067                         estr = "PCA_TLP_ABORT"; break;
2068                 default:
2069                         goto error;
2070                 }
2071                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2072                         estr, fault.func ? "VF" : "PF", fault.func,
2073                         fault.address, fault.specinfo);
2074         }
2075
2076         /* Process THI fault */
2077         if (eicr & FM10K_EICR_THI_FAULT) {
2078                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2079                 if (err)
2080                         goto error;
2081                 switch (fault.type) {
2082                 case THI_NO_FAULT:
2083                         estr = "THI_NO_FAULT"; break;
2084                 case THI_MAL_DIS_Q_FAULT:
2085                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2086                 default:
2087                         goto error;
2088                 }
2089                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2090                         estr, fault.func ? "VF" : "PF", fault.func,
2091                         fault.address, fault.specinfo);
2092         }
2093
2094         /* Process FUM fault */
2095         if (eicr & FM10K_EICR_FUM_FAULT) {
2096                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2097                 if (err)
2098                         goto error;
2099                 switch (fault.type) {
2100                 case FUM_NO_FAULT:
2101                         estr = "FUM_NO_FAULT"; break;
2102                 case FUM_UNMAPPED_ADDR:
2103                         estr = "FUM_UNMAPPED_ADDR"; break;
2104                 case FUM_POISONED_TLP:
2105                         estr = "FUM_POISONED_TLP"; break;
2106                 case FUM_BAD_VF_QACCESS:
2107                         estr = "FUM_BAD_VF_QACCESS"; break;
2108                 case FUM_ADD_DECODE_ERR:
2109                         estr = "FUM_ADD_DECODE_ERR"; break;
2110                 case FUM_RO_ERROR:
2111                         estr = "FUM_RO_ERROR"; break;
2112                 case FUM_QPRC_CRC_ERROR:
2113                         estr = "FUM_QPRC_CRC_ERROR"; break;
2114                 case FUM_CSR_TIMEOUT:
2115                         estr = "FUM_CSR_TIMEOUT"; break;
2116                 case FUM_INVALID_TYPE:
2117                         estr = "FUM_INVALID_TYPE"; break;
2118                 case FUM_INVALID_LENGTH:
2119                         estr = "FUM_INVALID_LENGTH"; break;
2120                 case FUM_INVALID_BE:
2121                         estr = "FUM_INVALID_BE"; break;
2122                 case FUM_INVALID_ALIGN:
2123                         estr = "FUM_INVALID_ALIGN"; break;
2124                 default:
2125                         goto error;
2126                 }
2127                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2128                         estr, fault.func ? "VF" : "PF", fault.func,
2129                         fault.address, fault.specinfo);
2130         }
2131
2132         return 0;
2133 error:
2134         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2135         return err;
2136 }
2137
2138 /**
2139  * PF interrupt handler triggered by NIC for handling specific interrupt.
2140  *
2141  * @param handle
2142  *  Pointer to interrupt handle.
2143  * @param param
2144  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2145  *
2146  * @return
2147  *  void
2148  */
2149 static void
2150 fm10k_dev_interrupt_handler_pf(
2151                         __rte_unused struct rte_intr_handle *handle,
2152                         void *param)
2153 {
2154         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2155         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2156         uint32_t cause, status;
2157
2158         if (hw->mac.type != fm10k_mac_pf)
2159                 return;
2160
2161         cause = FM10K_READ_REG(hw, FM10K_EICR);
2162
2163         /* Handle PCI fault cases */
2164         if (cause & FM10K_EICR_FAULT_MASK) {
2165                 PMD_INIT_LOG(ERR, "INT: find fault!");
2166                 fm10k_dev_handle_fault(hw, cause);
2167         }
2168
2169         /* Handle switch up/down */
2170         if (cause & FM10K_EICR_SWITCHNOTREADY)
2171                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2172
2173         if (cause & FM10K_EICR_SWITCHREADY)
2174                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2175
2176         /* Handle mailbox message */
2177         fm10k_mbx_lock(hw);
2178         hw->mbx.ops.process(hw, &hw->mbx);
2179         fm10k_mbx_unlock(hw);
2180
2181         /* Handle SRAM error */
2182         if (cause & FM10K_EICR_SRAMERROR) {
2183                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2184
2185                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2186                 /* Write to clear pending bits */
2187                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2188
2189                 /* Todo: print out error message after shared code  updates */
2190         }
2191
2192         /* Clear these 3 events if having any */
2193         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2194                  FM10K_EICR_SWITCHREADY;
2195         if (cause)
2196                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2197
2198         /* Re-enable interrupt from device side */
2199         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2200                                         FM10K_ITR_MASK_CLEAR);
2201         /* Re-enable interrupt from host side */
2202         rte_intr_enable(&(dev->pci_dev->intr_handle));
2203 }
2204
2205 /**
2206  * VF interrupt handler triggered by NIC for handling specific interrupt.
2207  *
2208  * @param handle
2209  *  Pointer to interrupt handle.
2210  * @param param
2211  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2212  *
2213  * @return
2214  *  void
2215  */
2216 static void
2217 fm10k_dev_interrupt_handler_vf(
2218                         __rte_unused struct rte_intr_handle *handle,
2219                         void *param)
2220 {
2221         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2222         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2223
2224         if (hw->mac.type != fm10k_mac_vf)
2225                 return;
2226
2227         /* Handle mailbox message if lock is acquired */
2228         fm10k_mbx_lock(hw);
2229         hw->mbx.ops.process(hw, &hw->mbx);
2230         fm10k_mbx_unlock(hw);
2231
2232         /* Re-enable interrupt from device side */
2233         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2234                                         FM10K_ITR_MASK_CLEAR);
2235         /* Re-enable interrupt from host side */
2236         rte_intr_enable(&(dev->pci_dev->intr_handle));
2237 }
2238
2239 /* Mailbox message handler in VF */
2240 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2241         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2242         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2243         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2244         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2245 };
2246
2247 /* Mailbox message handler in PF */
2248 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2249         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2250         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2251         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2252         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2253         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2254         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2255         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2256 };
2257
2258 static int
2259 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2260 {
2261         int err;
2262
2263         /* Initialize mailbox lock */
2264         fm10k_mbx_initlock(hw);
2265
2266         /* Replace default message handler with new ones */
2267         if (hw->mac.type == fm10k_mac_pf)
2268                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2269         else
2270                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2271
2272         if (err) {
2273                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2274                                 err);
2275                 return err;
2276         }
2277         /* Connect to SM for PF device or PF for VF device */
2278         return hw->mbx.ops.connect(hw, &hw->mbx);
2279 }
2280
2281 static void
2282 fm10k_close_mbx_service(struct fm10k_hw *hw)
2283 {
2284         /* Disconnect from SM for PF device or PF for VF device */
2285         hw->mbx.ops.disconnect(hw, &hw->mbx);
2286 }
2287
2288 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2289         .dev_configure          = fm10k_dev_configure,
2290         .dev_start              = fm10k_dev_start,
2291         .dev_stop               = fm10k_dev_stop,
2292         .dev_close              = fm10k_dev_close,
2293         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2294         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2295         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2296         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2297         .stats_get              = fm10k_stats_get,
2298         .stats_reset            = fm10k_stats_reset,
2299         .link_update            = fm10k_link_update,
2300         .dev_infos_get          = fm10k_dev_infos_get,
2301         .vlan_filter_set        = fm10k_vlan_filter_set,
2302         .vlan_offload_set       = fm10k_vlan_offload_set,
2303         .mac_addr_add           = fm10k_macaddr_add,
2304         .mac_addr_remove        = fm10k_macaddr_remove,
2305         .rx_queue_start         = fm10k_dev_rx_queue_start,
2306         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2307         .tx_queue_start         = fm10k_dev_tx_queue_start,
2308         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2309         .rx_queue_setup         = fm10k_rx_queue_setup,
2310         .rx_queue_release       = fm10k_rx_queue_release,
2311         .tx_queue_setup         = fm10k_tx_queue_setup,
2312         .tx_queue_release       = fm10k_tx_queue_release,
2313         .reta_update            = fm10k_reta_update,
2314         .reta_query             = fm10k_reta_query,
2315         .rss_hash_update        = fm10k_rss_hash_update,
2316         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2317 };
2318
2319 static void __attribute__((cold))
2320 fm10k_set_tx_function(struct rte_eth_dev *dev)
2321 {
2322         struct fm10k_tx_queue *txq;
2323         int i;
2324         int use_sse = 1;
2325
2326         for (i = 0; i < dev->data->nb_tx_queues; i++) {
2327                 txq = dev->data->tx_queues[i];
2328                 if ((txq->txq_flags & FM10K_SIMPLE_TX_FLAG) !=
2329                         FM10K_SIMPLE_TX_FLAG) {
2330                         use_sse = 0;
2331                         break;
2332                 }
2333         }
2334
2335         if (use_sse) {
2336                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2337                         txq = dev->data->tx_queues[i];
2338                         fm10k_txq_vec_setup(txq);
2339                 }
2340                 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2341         } else
2342                 dev->tx_pkt_burst = fm10k_xmit_pkts;
2343 }
2344
2345 static void __attribute__((cold))
2346 fm10k_set_rx_function(struct rte_eth_dev *dev)
2347 {
2348         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2349         uint16_t i, rx_using_sse;
2350
2351         /* In order to allow Vector Rx there are a few configuration
2352          * conditions to be met.
2353          */
2354         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2355                 if (dev->data->scattered_rx)
2356                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2357                 else
2358                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2359         } else if (dev->data->scattered_rx)
2360                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2361
2362         rx_using_sse =
2363                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2364                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2365
2366         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2367                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2368
2369                 rxq->rx_using_sse = rx_using_sse;
2370         }
2371 }
2372
2373 static void
2374 fm10k_params_init(struct rte_eth_dev *dev)
2375 {
2376         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2377         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2378
2379         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2380          * there is no way to get link status without reading BAR4.  Until this
2381          * works, assume we have maximum bandwidth.
2382          * @todo - fix bus info
2383          */
2384         hw->bus_caps.speed = fm10k_bus_speed_8000;
2385         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2386         hw->bus_caps.payload = fm10k_bus_payload_512;
2387         hw->bus.speed = fm10k_bus_speed_8000;
2388         hw->bus.width = fm10k_bus_width_pcie_x8;
2389         hw->bus.payload = fm10k_bus_payload_256;
2390
2391         info->rx_vec_allowed = true;
2392 }
2393
2394 static int
2395 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2396 {
2397         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2398         int diag;
2399         struct fm10k_macvlan_filter_info *macvlan;
2400
2401         PMD_INIT_FUNC_TRACE();
2402
2403         dev->dev_ops = &fm10k_eth_dev_ops;
2404         dev->rx_pkt_burst = &fm10k_recv_pkts;
2405         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2406
2407         /* only initialize in the primary process */
2408         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2409                 return 0;
2410
2411         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2412         memset(macvlan, 0, sizeof(*macvlan));
2413         /* Vendor and Device ID need to be set before init of shared code */
2414         memset(hw, 0, sizeof(*hw));
2415         hw->device_id = dev->pci_dev->id.device_id;
2416         hw->vendor_id = dev->pci_dev->id.vendor_id;
2417         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2418         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2419         hw->revision_id = 0;
2420         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2421         if (hw->hw_addr == NULL) {
2422                 PMD_INIT_LOG(ERR, "Bad mem resource."
2423                         " Try to blacklist unused devices.");
2424                 return -EIO;
2425         }
2426
2427         /* Store fm10k_adapter pointer */
2428         hw->back = dev->data->dev_private;
2429
2430         /* Initialize the shared code */
2431         diag = fm10k_init_shared_code(hw);
2432         if (diag != FM10K_SUCCESS) {
2433                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2434                 return -EIO;
2435         }
2436
2437         /* Initialize parameters */
2438         fm10k_params_init(dev);
2439
2440         /* Initialize the hw */
2441         diag = fm10k_init_hw(hw);
2442         if (diag != FM10K_SUCCESS) {
2443                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2444                 return -EIO;
2445         }
2446
2447         /* Initialize MAC address(es) */
2448         dev->data->mac_addrs = rte_zmalloc("fm10k",
2449                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2450         if (dev->data->mac_addrs == NULL) {
2451                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2452                 return -ENOMEM;
2453         }
2454
2455         diag = fm10k_read_mac_addr(hw);
2456
2457         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2458                         &dev->data->mac_addrs[0]);
2459
2460         if (diag != FM10K_SUCCESS ||
2461                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2462
2463                 /* Generate a random addr */
2464                 eth_random_addr(hw->mac.addr);
2465                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2466                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2467                 &dev->data->mac_addrs[0]);
2468         }
2469
2470         /* Reset the hw statistics */
2471         fm10k_stats_reset(dev);
2472
2473         /* Reset the hw */
2474         diag = fm10k_reset_hw(hw);
2475         if (diag != FM10K_SUCCESS) {
2476                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2477                 return -EIO;
2478         }
2479
2480         /* Setup mailbox service */
2481         diag = fm10k_setup_mbx_service(hw);
2482         if (diag != FM10K_SUCCESS) {
2483                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2484                 return -EIO;
2485         }
2486
2487         /*PF/VF has different interrupt handling mechanism */
2488         if (hw->mac.type == fm10k_mac_pf) {
2489                 /* register callback func to eal lib */
2490                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2491                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2492
2493                 /* enable MISC interrupt */
2494                 fm10k_dev_enable_intr_pf(dev);
2495         } else { /* VF */
2496                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2497                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2498
2499                 fm10k_dev_enable_intr_vf(dev);
2500         }
2501
2502         /* Enable uio intr after callback registered */
2503         rte_intr_enable(&(dev->pci_dev->intr_handle));
2504
2505         hw->mac.ops.update_int_moderator(hw);
2506
2507         /* Make sure Switch Manager is ready before going forward. */
2508         if (hw->mac.type == fm10k_mac_pf) {
2509                 int switch_ready = 0;
2510                 int i;
2511
2512                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2513                         fm10k_mbx_lock(hw);
2514                         hw->mac.ops.get_host_state(hw, &switch_ready);
2515                         fm10k_mbx_unlock(hw);
2516                         if (switch_ready)
2517                                 break;
2518                         /* Delay some time to acquire async LPORT_MAP info. */
2519                         rte_delay_us(WAIT_SWITCH_MSG_US);
2520                 }
2521
2522                 if (switch_ready == 0) {
2523                         PMD_INIT_LOG(ERR, "switch is not ready");
2524                         return -1;
2525                 }
2526         }
2527
2528         /*
2529          * Below function will trigger operations on mailbox, acquire lock to
2530          * avoid race condition from interrupt handler. Operations on mailbox
2531          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2532          * will handle and generate an interrupt to our side. Then,  FIFO in
2533          * mailbox will be touched.
2534          */
2535         fm10k_mbx_lock(hw);
2536         /* Enable port first */
2537         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2538
2539         /* Set unicast mode by default. App can change to other mode in other
2540          * API func.
2541          */
2542         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2543                                         FM10K_XCAST_MODE_NONE);
2544
2545         fm10k_mbx_unlock(hw);
2546
2547         /* Add default mac address */
2548         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2549                 MAIN_VSI_POOL_NUMBER);
2550
2551         return 0;
2552 }
2553
2554 static int
2555 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2556 {
2557         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2558
2559         PMD_INIT_FUNC_TRACE();
2560
2561         /* only uninitialize in the primary process */
2562         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2563                 return 0;
2564
2565         /* safe to close dev here */
2566         fm10k_dev_close(dev);
2567
2568         dev->dev_ops = NULL;
2569         dev->rx_pkt_burst = NULL;
2570         dev->tx_pkt_burst = NULL;
2571
2572         /* disable uio/vfio intr */
2573         rte_intr_disable(&(dev->pci_dev->intr_handle));
2574
2575         /*PF/VF has different interrupt handling mechanism */
2576         if (hw->mac.type == fm10k_mac_pf) {
2577                 /* disable interrupt */
2578                 fm10k_dev_disable_intr_pf(dev);
2579
2580                 /* unregister callback func to eal lib */
2581                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2582                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2583         } else {
2584                 /* disable interrupt */
2585                 fm10k_dev_disable_intr_vf(dev);
2586
2587                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2588                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2589         }
2590
2591         /* free mac memory */
2592         if (dev->data->mac_addrs) {
2593                 rte_free(dev->data->mac_addrs);
2594                 dev->data->mac_addrs = NULL;
2595         }
2596
2597         memset(hw, 0, sizeof(*hw));
2598
2599         return 0;
2600 }
2601
2602 /*
2603  * The set of PCI devices this driver supports. This driver will enable both PF
2604  * and SRIOV-VF devices.
2605  */
2606 static const struct rte_pci_id pci_id_fm10k_map[] = {
2607 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2608 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2609 #include "rte_pci_dev_ids.h"
2610         { .vendor_id = 0, /* sentinel */ },
2611 };
2612
2613 static struct eth_driver rte_pmd_fm10k = {
2614         .pci_drv = {
2615                 .name = "rte_pmd_fm10k",
2616                 .id_table = pci_id_fm10k_map,
2617                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2618         },
2619         .eth_dev_init = eth_fm10k_dev_init,
2620         .eth_dev_uninit = eth_fm10k_dev_uninit,
2621         .dev_private_size = sizeof(struct fm10k_adapter),
2622 };
2623
2624 /*
2625  * Driver initialization routine.
2626  * Invoked once at EAL init time.
2627  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2628  */
2629 static int
2630 rte_pmd_fm10k_init(__rte_unused const char *name,
2631         __rte_unused const char *params)
2632 {
2633         PMD_INIT_FUNC_TRACE();
2634         rte_eth_driver_register(&rte_pmd_fm10k);
2635         return 0;
2636 }
2637
2638 static struct rte_driver rte_fm10k_driver = {
2639         .type = PMD_PDEV,
2640         .init = rte_pmd_fm10k_init,
2641 };
2642
2643 PMD_REGISTER_DRIVER(rte_fm10k_driver);