drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  64 static int
  65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  67         const u8 *mac, bool add, uint32_t pool);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         return 0;
 125 }
 126
 127 /*
 128  * clean queue, descriptor rings, free software buffers used when stopping
 129  * device.
 130  */
 131 static inline void
 132 rx_queue_clean(struct fm10k_rx_queue *q)
 133 {
 134         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 135         uint32_t i;
 136         PMD_INIT_FUNC_TRACE();
 137
 138         /* zero descriptor rings */
 139         for (i = 0; i < q->nb_desc; ++i)
 140                 q->hw_ring[i] = zero;
 141
 142         /* free software buffers */
 143         for (i = 0; i < q->nb_desc; ++i) {
 144                 if (q->sw_ring[i]) {
 145                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 146                         q->sw_ring[i] = NULL;
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * free all queue memory used when releasing the queue (i.e. configure)
 153  */
 154 static inline void
 155 rx_queue_free(struct fm10k_rx_queue *q)
 156 {
 157         PMD_INIT_FUNC_TRACE();
 158         if (q) {
 159                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 160                 rx_queue_clean(q);
 161                 if (q->sw_ring) {
 162                         rte_free(q->sw_ring);
 163                         q->sw_ring = NULL;
 164                 }
 165                 rte_free(q);
 166                 q = NULL;
 167         }
 168 }
 169
 170 /*
 171  * disable RX queue, wait unitl HW finished necessary flush operation
 172  */
 173 static inline int
 174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 175 {
 176         uint32_t reg, i;
 177
 178         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 179         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 180                         reg & ~FM10K_RXQCTL_ENABLE);
 181
 182         /* Wait 100us at most */
 183         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 184                 rte_delay_us(1);
 185                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 186                 if (!(reg & FM10K_RXQCTL_ENABLE))
 187                         break;
 188         }
 189
 190         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 191                 return -1;
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * reset queue to initial state, allocate software buffers used when starting
 198  * device
 199  */
 200 static inline void
 201 tx_queue_reset(struct fm10k_tx_queue *q)
 202 {
 203         PMD_INIT_FUNC_TRACE();
 204         q->last_free = 0;
 205         q->next_free = 0;
 206         q->nb_used = 0;
 207         q->nb_free = q->nb_desc - 1;
 208         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 209         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 210 }
 211
 212 /*
 213  * clean queue, descriptor rings, free software buffers used when stopping
 214  * device
 215  */
 216 static inline void
 217 tx_queue_clean(struct fm10k_tx_queue *q)
 218 {
 219         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 220         uint32_t i;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         /* zero descriptor rings */
 224         for (i = 0; i < q->nb_desc; ++i)
 225                 q->hw_ring[i] = zero;
 226
 227         /* free software buffers */
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 if (q->sw_ring[i]) {
 230                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 231                         q->sw_ring[i] = NULL;
 232                 }
 233         }
 234 }
 235
 236 /*
 237  * free all queue memory used when releasing the queue (i.e. configure)
 238  */
 239 static inline void
 240 tx_queue_free(struct fm10k_tx_queue *q)
 241 {
 242         PMD_INIT_FUNC_TRACE();
 243         if (q) {
 244                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 245                 tx_queue_clean(q);
 246                 if (q->rs_tracker.list) {
 247                         rte_free(q->rs_tracker.list);
 248                         q->rs_tracker.list = NULL;
 249                 }
 250                 if (q->sw_ring) {
 251                         rte_free(q->sw_ring);
 252                         q->sw_ring = NULL;
 253                 }
 254                 rte_free(q);
 255                 q = NULL;
 256         }
 257 }
 258
 259 /*
 260  * disable TX queue, wait unitl HW finished necessary flush operation
 261  */
 262 static inline int
 263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 264 {
 265         uint32_t reg, i;
 266
 267         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 268         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 269                         reg & ~FM10K_TXDCTL_ENABLE);
 270
 271         /* Wait 100us at most */
 272         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 273                 rte_delay_us(1);
 274                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 275                 if (!(reg & FM10K_TXDCTL_ENABLE))
 276                         break;
 277         }
 278
 279         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 280                 return -1;
 281
 282         return 0;
 283 }
 284
 285 static int
 286 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 287 {
 288         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 291         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 292
 293         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 294
 295         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 296                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 297                 return -EINVAL;
 298         }
 299
 300         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 301                 return 0;
 302
 303         if (hw->mac.type == fm10k_mac_vf) {
 304                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 305                 return -EINVAL;
 306         }
 307
 308         /* Check VMDQ queue pool number */
 309         if (vmdq_conf->nb_queue_pools >
 310                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 311                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 312                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 313                         vmdq_conf->nb_queue_pools);
 314                 return -EINVAL;
 315         }
 316
 317         return 0;
 318 }
 319
 320 static int
 321 fm10k_dev_configure(struct rte_eth_dev *dev)
 322 {
 323         int ret;
 324
 325         PMD_INIT_FUNC_TRACE();
 326
 327         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 328                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 329         /* multipe queue mode checking */
 330         ret  = fm10k_check_mq_mode(dev);
 331         if (ret != 0) {
 332                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 333                             ret);
 334                 return ret;
 335         }
 336
 337         return 0;
 338 }
 339
 340 /* fls = find last set bit = 32 minus the number of leading zeros */
 341 #ifndef fls
 342 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 343 #endif
 344
 345 static void
 346 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 347 {
 348         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 349         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 350         uint32_t i;
 351
 352         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 353
 354         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 355                 if (!vmdq_conf->pool_map[i].pools)
 356                         continue;
 357                 fm10k_mbx_lock(hw);
 358                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 359                 fm10k_mbx_unlock(hw);
 360         }
 361 }
 362
 363 static void
 364 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 365 {
 366         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 367
 368         /* Add default mac address */
 369         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 370                 MAIN_VSI_POOL_NUMBER);
 371 }
 372
 373 static void
 374 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 375 {
 376         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 377         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 378         uint32_t mrqc, *key, i, reta, j;
 379         uint64_t hf;
 380
 381 #define RSS_KEY_SIZE 40
 382         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 383                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 384                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 385                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 386                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 387                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 388         };
 389
 390         if (dev->data->nb_rx_queues == 1 ||
 391             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 392             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 393                 return;
 394
 395         /* random key is rss_intel_key (default) or user provided (rss_key) */
 396         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 397                 key = (uint32_t *)rss_intel_key;
 398         else
 399                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 400
 401         /* Now fill our hash function seeds, 4 bytes at a time */
 402         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 403                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 404
 405         /*
 406          * Fill in redirection table
 407          * The byte-swap is needed because NIC registers are in
 408          * little-endian order.
 409          */
 410         reta = 0;
 411         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 412                 if (j == dev->data->nb_rx_queues)
 413                         j = 0;
 414                 reta = (reta << CHAR_BIT) | j;
 415                 if ((i & 3) == 3)
 416                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 417                                         rte_bswap32(reta));
 418         }
 419
 420         /*
 421          * Generate RSS hash based on packet types, TCP/UDP
 422          * port numbers and/or IPv4/v6 src and dst addresses
 423          */
 424         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 425         mrqc = 0;
 426         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 427         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 428         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 429         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 430         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 431         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 432         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 433         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 434         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 435
 436         if (mrqc == 0) {
 437                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 438                         "supported", hf);
 439                 return;
 440         }
 441
 442         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 443 }
 444
 445 static void
 446 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 447         uint16_t nb_lport_old, uint16_t nb_lport_new)
 448 {
 449         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 450         uint32_t i;
 451
 452         fm10k_mbx_lock(hw);
 453         /* Disable previous logic ports */
 454         if (nb_lport_old)
 455                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 456                         nb_lport_old, false);
 457         /* Enable new logic ports */
 458         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 459                 nb_lport_new, true);
 460         fm10k_mbx_unlock(hw);
 461
 462         for (i = 0; i < nb_lport_new; i++) {
 463                 /* Set unicast mode by default. App can change
 464                  * to other mode in other API func.
 465                  */
 466                 fm10k_mbx_lock(hw);
 467                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 468                         FM10K_XCAST_MODE_NONE);
 469                 fm10k_mbx_unlock(hw);
 470         }
 471 }
 472
 473 static void
 474 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 475 {
 476         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 477         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 478         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 479         struct fm10k_macvlan_filter_info *macvlan;
 480         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 481         uint16_t nb_lport_new, nb_lport_old;
 482
 483         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 484         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 485
 486         fm10k_dev_rss_configure(dev);
 487
 488         /* only PF supports VMDQ */
 489         if (hw->mac.type != fm10k_mac_pf)
 490                 return;
 491
 492         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 493                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 494
 495         /* no pool number change, no need to update logic port and VLAN/MAC */
 496         if (macvlan->nb_queue_pools == nb_queue_pools)
 497                 return;
 498
 499         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 500         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 501         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 502
 503         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 504         memset(dev->data->mac_addrs, 0,
 505                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 506         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 507                 &dev->data->mac_addrs[0]);
 508         memset(macvlan, 0, sizeof(*macvlan));
 509         macvlan->nb_queue_pools = nb_queue_pools;
 510
 511         if (nb_queue_pools)
 512                 fm10k_dev_vmdq_rx_configure(dev);
 513         else
 514                 fm10k_dev_pf_main_vsi_reset(dev);
 515 }
 516
 517 static int
 518 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 519 {
 520         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 521         int i, ret;
 522         struct fm10k_tx_queue *txq;
 523         uint64_t base_addr;
 524         uint32_t size;
 525
 526         /* Disable TXINT to avoid possible interrupt */
 527         for (i = 0; i < hw->mac.max_queues; i++)
 528                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 529                                 3 << FM10K_TXINT_TIMER_SHIFT);
 530
 531         /* Setup TX queue */
 532         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 533                 txq = dev->data->tx_queues[i];
 534                 base_addr = txq->hw_ring_phys_addr;
 535                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 536
 537                 /* disable queue to avoid issues while updating state */
 538                 ret = tx_queue_disable(hw, i);
 539                 if (ret) {
 540                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 541                         return -1;
 542                 }
 543
 544                 /* set location and size for descriptor ring */
 545                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 546                                 base_addr & UINT64_LOWER_32BITS_MASK);
 547                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 548                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 549                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 550         }
 551         return 0;
 552 }
 553
 554 static int
 555 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 556 {
 557         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 558         int i, ret;
 559         struct fm10k_rx_queue *rxq;
 560         uint64_t base_addr;
 561         uint32_t size;
 562         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 563         uint16_t buf_size;
 564
 565         /* Disable RXINT to avoid possible interrupt */
 566         for (i = 0; i < hw->mac.max_queues; i++)
 567                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 568                                 3 << FM10K_RXINT_TIMER_SHIFT);
 569
 570         /* Setup RX queues */
 571         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 572                 rxq = dev->data->rx_queues[i];
 573                 base_addr = rxq->hw_ring_phys_addr;
 574                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 575
 576                 /* disable queue to avoid issues while updating state */
 577                 ret = rx_queue_disable(hw, i);
 578                 if (ret) {
 579                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 580                         return -1;
 581                 }
 582
 583                 /* Setup the Base and Length of the Rx Descriptor Ring */
 584                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 585                                 base_addr & UINT64_LOWER_32BITS_MASK);
 586                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 587                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 588                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 589
 590                 /* Configure the Rx buffer size for one buff without split */
 591                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 592                         RTE_PKTMBUF_HEADROOM);
 593                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 594                  * reserved for this purpose, and the worst case could be 511B.
 595                  * But SRR reg assumes all buffers have the same size. In order
 596                  * to fill the gap, we'll have to consider the worst case and
 597                  * assume 512B is reserved. If we don't do so, it's possible
 598                  * for HW to overwrite data to next mbuf.
 599                  */
 600                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 601
 602                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 603                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 604
 605                 /* It adds dual VLAN length for supporting dual VLAN */
 606                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 607                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 608                         dev->data->dev_conf.rxmode.enable_scatter) {
 609                         uint32_t reg;
 610                         dev->data->scattered_rx = 1;
 611                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 612                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 613                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 614                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 615                 }
 616
 617                 /* Enable drop on empty, it's RO for VF */
 618                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 619                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 620
 621                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 622                 FM10K_WRITE_FLUSH(hw);
 623         }
 624
 625         /* Configure VMDQ/RSS if applicable */
 626         fm10k_dev_mq_rx_configure(dev);
 627         return 0;
 628 }
 629
 630 static int
 631 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 632 {
 633         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 634         int err = -1;
 635         uint32_t reg;
 636         struct fm10k_rx_queue *rxq;
 637
 638         PMD_INIT_FUNC_TRACE();
 639
 640         if (rx_queue_id < dev->data->nb_rx_queues) {
 641                 rxq = dev->data->rx_queues[rx_queue_id];
 642                 err = rx_queue_reset(rxq);
 643                 if (err == -ENOMEM) {
 644                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 645                         return err;
 646                 } else if (err == -EINVAL) {
 647                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 648                                 " %d", err);
 649                         return err;
 650                 }
 651
 652                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 653                  * Note: this must be done AFTER the queue is enabled on real
 654                  * hardware, but BEFORE the queue is enabled when using the
 655                  * emulation platform. Do it in both places for now and remove
 656                  * this comment and the following two register writes when the
 657                  * emulation platform is no longer being used.
 658                  */
 659                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 660                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 661
 662                 /* Set PF ownership flag for PF devices */
 663                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 664                 if (hw->mac.type == fm10k_mac_pf)
 665                         reg |= FM10K_RXQCTL_PF;
 666                 reg |= FM10K_RXQCTL_ENABLE;
 667                 /* enable RX queue */
 668                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 669                 FM10K_WRITE_FLUSH(hw);
 670
 671                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 672                  * Note: this must be done AFTER the queue is enabled
 673                  */
 674                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 675                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 676         }
 677
 678         return err;
 679 }
 680
 681 static int
 682 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 683 {
 684         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 685
 686         PMD_INIT_FUNC_TRACE();
 687
 688         if (rx_queue_id < dev->data->nb_rx_queues) {
 689                 /* Disable RX queue */
 690                 rx_queue_disable(hw, rx_queue_id);
 691
 692                 /* Free mbuf and clean HW ring */
 693                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 694         }
 695
 696         return 0;
 697 }
 698
 699 static int
 700 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 701 {
 702         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 703         /** @todo - this should be defined in the shared code */
 704 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 705         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 706         int err = 0;
 707
 708         PMD_INIT_FUNC_TRACE();
 709
 710         if (tx_queue_id < dev->data->nb_tx_queues) {
 711                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 712
 713                 /* reset head and tail pointers */
 714                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 715                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 716
 717                 /* enable TX queue */
 718                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 719                                         FM10K_TXDCTL_ENABLE | txdctl);
 720                 FM10K_WRITE_FLUSH(hw);
 721         } else
 722                 err = -1;
 723
 724         return err;
 725 }
 726
 727 static int
 728 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 729 {
 730         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 731
 732         PMD_INIT_FUNC_TRACE();
 733
 734         if (tx_queue_id < dev->data->nb_tx_queues) {
 735                 tx_queue_disable(hw, tx_queue_id);
 736                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 737         }
 738
 739         return 0;
 740 }
 741
 742 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 743 {
 744         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 745                 != FM10K_DGLORTMAP_NONE);
 746 }
 747
 748 static void
 749 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 750 {
 751         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 752         int status;
 753
 754         PMD_INIT_FUNC_TRACE();
 755
 756         /* Return if it didn't acquire valid glort range */
 757         if (!fm10k_glort_valid(hw))
 758                 return;
 759
 760         fm10k_mbx_lock(hw);
 761         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 762                                 FM10K_XCAST_MODE_PROMISC);
 763         fm10k_mbx_unlock(hw);
 764
 765         if (status != FM10K_SUCCESS)
 766                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 767 }
 768
 769 static void
 770 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 771 {
 772         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 773         uint8_t mode;
 774         int status;
 775
 776         PMD_INIT_FUNC_TRACE();
 777
 778         /* Return if it didn't acquire valid glort range */
 779         if (!fm10k_glort_valid(hw))
 780                 return;
 781
 782         if (dev->data->all_multicast == 1)
 783                 mode = FM10K_XCAST_MODE_ALLMULTI;
 784         else
 785                 mode = FM10K_XCAST_MODE_NONE;
 786
 787         fm10k_mbx_lock(hw);
 788         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 789                                 mode);
 790         fm10k_mbx_unlock(hw);
 791
 792         if (status != FM10K_SUCCESS)
 793                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 794 }
 795
 796 static void
 797 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 798 {
 799         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 800         int status;
 801
 802         PMD_INIT_FUNC_TRACE();
 803
 804         /* Return if it didn't acquire valid glort range */
 805         if (!fm10k_glort_valid(hw))
 806                 return;
 807
 808         /* If promiscuous mode is enabled, it doesn't make sense to enable
 809          * allmulticast and disable promiscuous since fm10k only can select
 810          * one of the modes.
 811          */
 812         if (dev->data->promiscuous) {
 813                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 814                         "needn't enable allmulticast");
 815                 return;
 816         }
 817
 818         fm10k_mbx_lock(hw);
 819         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 820                                 FM10K_XCAST_MODE_ALLMULTI);
 821         fm10k_mbx_unlock(hw);
 822
 823         if (status != FM10K_SUCCESS)
 824                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 825 }
 826
 827 static void
 828 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 829 {
 830         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 831         int status;
 832
 833         PMD_INIT_FUNC_TRACE();
 834
 835         /* Return if it didn't acquire valid glort range */
 836         if (!fm10k_glort_valid(hw))
 837                 return;
 838
 839         if (dev->data->promiscuous) {
 840                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 841                         "since promisc mode is enabled");
 842                 return;
 843         }
 844
 845         fm10k_mbx_lock(hw);
 846         /* Change mode to unicast mode */
 847         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 848                                 FM10K_XCAST_MODE_NONE);
 849         fm10k_mbx_unlock(hw);
 850
 851         if (status != FM10K_SUCCESS)
 852                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 853 }
 854
 855 static void
 856 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 857 {
 858         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859         uint32_t dglortdec, pool_len, rss_len, i;
 860         uint16_t nb_queue_pools;
 861         struct fm10k_macvlan_filter_info *macvlan;
 862
 863         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 864         nb_queue_pools = macvlan->nb_queue_pools;
 865         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 866         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 867         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 868
 869         /* Establish only MAP 0 as valid */
 870         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 871
 872         /* Configure VMDQ/RSS DGlort Decoder */
 873         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 874
 875         /* Invalidate all other GLORT entries */
 876         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 877                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 878                                 FM10K_DGLORTMAP_NONE);
 879 }
 880
 881 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 882 static int
 883 fm10k_dev_start(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886         int i, diag;
 887
 888         PMD_INIT_FUNC_TRACE();
 889
 890         /* stop, init, then start the hw */
 891         diag = fm10k_stop_hw(hw);
 892         if (diag != FM10K_SUCCESS) {
 893                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 894                 return -EIO;
 895         }
 896
 897         diag = fm10k_init_hw(hw);
 898         if (diag != FM10K_SUCCESS) {
 899                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 900                 return -EIO;
 901         }
 902
 903         diag = fm10k_start_hw(hw);
 904         if (diag != FM10K_SUCCESS) {
 905                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 906                 return -EIO;
 907         }
 908
 909         diag = fm10k_dev_tx_init(dev);
 910         if (diag) {
 911                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 912                 return diag;
 913         }
 914
 915         diag = fm10k_dev_rx_init(dev);
 916         if (diag) {
 917                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 918                 return diag;
 919         }
 920
 921         if (hw->mac.type == fm10k_mac_pf)
 922                 fm10k_dev_dglort_map_configure(dev);
 923
 924         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 925                 struct fm10k_rx_queue *rxq;
 926                 rxq = dev->data->rx_queues[i];
 927
 928                 if (rxq->rx_deferred_start)
 929                         continue;
 930                 diag = fm10k_dev_rx_queue_start(dev, i);
 931                 if (diag != 0) {
 932                         int j;
 933                         for (j = 0; j < i; ++j)
 934                                 rx_queue_clean(dev->data->rx_queues[j]);
 935                         return diag;
 936                 }
 937         }
 938
 939         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 940                 struct fm10k_tx_queue *txq;
 941                 txq = dev->data->tx_queues[i];
 942
 943                 if (txq->tx_deferred_start)
 944                         continue;
 945                 diag = fm10k_dev_tx_queue_start(dev, i);
 946                 if (diag != 0) {
 947                         int j;
 948                         for (j = 0; j < i; ++j)
 949                                 tx_queue_clean(dev->data->tx_queues[j]);
 950                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 951                                 rx_queue_clean(dev->data->rx_queues[j]);
 952                         return diag;
 953                 }
 954         }
 955
 956         /* Update default vlan when not in VMDQ mode */
 957         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 958                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 959
 960         return 0;
 961 }
 962
 963 static void
 964 fm10k_dev_stop(struct rte_eth_dev *dev)
 965 {
 966         int i;
 967
 968         PMD_INIT_FUNC_TRACE();
 969
 970         if (dev->data->tx_queues)
 971                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 972                         fm10k_dev_tx_queue_stop(dev, i);
 973
 974         if (dev->data->rx_queues)
 975                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 976                         fm10k_dev_rx_queue_stop(dev, i);
 977 }
 978
 979 static void
 980 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 981 {
 982         int i;
 983
 984         PMD_INIT_FUNC_TRACE();
 985
 986         if (dev->data->tx_queues) {
 987                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 988                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 989         }
 990
 991         if (dev->data->rx_queues) {
 992                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 993                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 994         }
 995 }
 996
 997 static void
 998 fm10k_dev_close(struct rte_eth_dev *dev)
 999 {
1000         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1001         uint16_t nb_lport;
1002         struct fm10k_macvlan_filter_info *macvlan;
1003
1004         PMD_INIT_FUNC_TRACE();
1005
1006         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1007         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1008         fm10k_mbx_lock(hw);
1009         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1010                 nb_lport, false);
1011         fm10k_mbx_unlock(hw);
1012
1013         /* Stop mailbox service first */
1014         fm10k_close_mbx_service(hw);
1015         fm10k_dev_stop(dev);
1016         fm10k_dev_queue_release(dev);
1017         fm10k_stop_hw(hw);
1018 }
1019
1020 static int
1021 fm10k_link_update(struct rte_eth_dev *dev,
1022         __rte_unused int wait_to_complete)
1023 {
1024         PMD_INIT_FUNC_TRACE();
1025
1026         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1027          * x8 PCIe interface. For now, we leave the speed undefined since there
1028          * is no 50Gbps Ethernet. */
1029         dev->data->dev_link.link_speed  = 0;
1030         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1031         dev->data->dev_link.link_status = 1;
1032
1033         return 0;
1034 }
1035
1036 static void
1037 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1038 {
1039         uint64_t ipackets, opackets, ibytes, obytes;
1040         struct fm10k_hw *hw =
1041                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1042         struct fm10k_hw_stats *hw_stats =
1043                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1044         int i;
1045
1046         PMD_INIT_FUNC_TRACE();
1047
1048         fm10k_update_hw_stats(hw, hw_stats);
1049
1050         ipackets = opackets = ibytes = obytes = 0;
1051         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1052                 (i < hw->mac.max_queues); ++i) {
1053                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1054                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1055                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1056                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1057                 ipackets += stats->q_ipackets[i];
1058                 opackets += stats->q_opackets[i];
1059                 ibytes   += stats->q_ibytes[i];
1060                 obytes   += stats->q_obytes[i];
1061         }
1062         stats->ipackets = ipackets;
1063         stats->opackets = opackets;
1064         stats->ibytes = ibytes;
1065         stats->obytes = obytes;
1066 }
1067
1068 static void
1069 fm10k_stats_reset(struct rte_eth_dev *dev)
1070 {
1071         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1072         struct fm10k_hw_stats *hw_stats =
1073                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         memset(hw_stats, 0, sizeof(*hw_stats));
1078         fm10k_rebind_hw_stats(hw, hw_stats);
1079 }
1080
1081 static void
1082 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1083         struct rte_eth_dev_info *dev_info)
1084 {
1085         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1086
1087         PMD_INIT_FUNC_TRACE();
1088
1089         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1090         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1091         dev_info->max_rx_queues      = hw->mac.max_queues;
1092         dev_info->max_tx_queues      = hw->mac.max_queues;
1093         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1094         dev_info->max_hash_mac_addrs = 0;
1095         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1096         dev_info->vmdq_pool_base     = 0;
1097         dev_info->vmdq_queue_base    = 0;
1098         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1099         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1100         dev_info->rx_offload_capa =
1101                 DEV_RX_OFFLOAD_VLAN_STRIP |
1102                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1103                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1104                 DEV_RX_OFFLOAD_TCP_CKSUM;
1105         dev_info->tx_offload_capa =
1106                 DEV_TX_OFFLOAD_VLAN_INSERT |
1107                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1108                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1109                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1110                 DEV_TX_OFFLOAD_TCP_TSO;
1111
1112         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1113         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1114
1115         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1116                 .rx_thresh = {
1117                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1118                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1119                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1120                 },
1121                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1122                 .rx_drop_en = 0,
1123         };
1124
1125         dev_info->default_txconf = (struct rte_eth_txconf) {
1126                 .tx_thresh = {
1127                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1128                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1129                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1130                 },
1131                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1132                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1133                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1134                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1135         };
1136
1137         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1138                 .nb_max = FM10K_MAX_RX_DESC,
1139                 .nb_min = FM10K_MIN_RX_DESC,
1140                 .nb_align = FM10K_MULT_RX_DESC,
1141         };
1142
1143         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1144                 .nb_max = FM10K_MAX_TX_DESC,
1145                 .nb_min = FM10K_MIN_TX_DESC,
1146                 .nb_align = FM10K_MULT_TX_DESC,
1147         };
1148 }
1149
1150 static int
1151 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1152 {
1153         s32 result;
1154         uint16_t mac_num = 0;
1155         uint32_t vid_idx, vid_bit, mac_index;
1156         struct fm10k_hw *hw;
1157         struct fm10k_macvlan_filter_info *macvlan;
1158         struct rte_eth_dev_data *data = dev->data;
1159
1160         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1161         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1162
1163         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1164                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1165                 return (-EINVAL);
1166         }
1167
1168         if (vlan_id > ETH_VLAN_ID_MAX) {
1169                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1170                 return (-EINVAL);
1171         }
1172
1173         vid_idx = FM10K_VFTA_IDX(vlan_id);
1174         vid_bit = FM10K_VFTA_BIT(vlan_id);
1175         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1176         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1177                 return 0;
1178         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1179         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1180                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1181                         "in the VLAN filter table");
1182                 return (-EINVAL);
1183         }
1184
1185         fm10k_mbx_lock(hw);
1186         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1187         fm10k_mbx_unlock(hw);
1188         if (result != FM10K_SUCCESS) {
1189                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1190                 return (-EIO);
1191         }
1192
1193         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1194                         (result == FM10K_SUCCESS); mac_index++) {
1195                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1196                         continue;
1197                 if (mac_num > macvlan->mac_num - 1) {
1198                         PMD_INIT_LOG(ERR, "MAC address number "
1199                                         "not match");
1200                         break;
1201                 }
1202                 fm10k_mbx_lock(hw);
1203                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1204                         data->mac_addrs[mac_index].addr_bytes,
1205                         vlan_id, on, 0);
1206                 fm10k_mbx_unlock(hw);
1207                 mac_num++;
1208         }
1209         if (result != FM10K_SUCCESS) {
1210                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1211                 return (-EIO);
1212         }
1213
1214         if (on) {
1215                 macvlan->vlan_num++;
1216                 macvlan->vfta[vid_idx] |= vid_bit;
1217         } else {
1218                 macvlan->vlan_num--;
1219                 macvlan->vfta[vid_idx] &= ~vid_bit;
1220         }
1221         return 0;
1222 }
1223
1224 static void
1225 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1226 {
1227         if (mask & ETH_VLAN_STRIP_MASK) {
1228                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1229                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1230                                         "always on in fm10k");
1231         }
1232
1233         if (mask & ETH_VLAN_EXTEND_MASK) {
1234                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1235                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1236                                         "supported in fm10k");
1237         }
1238
1239         if (mask & ETH_VLAN_FILTER_MASK) {
1240                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1241                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1242         }
1243 }
1244
1245 /* Add/Remove a MAC address, and update filters to main VSI */
1246 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1247                 const u8 *mac, bool add, uint32_t pool)
1248 {
1249         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1250         struct fm10k_macvlan_filter_info *macvlan;
1251         uint32_t i, j, k;
1252
1253         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1254
1255         if (pool != MAIN_VSI_POOL_NUMBER) {
1256                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1257                         "mac to pool %u", pool);
1258                 return;
1259         }
1260         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1261                 if (!macvlan->vfta[j])
1262                         continue;
1263                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1264                         if (!(macvlan->vfta[j] & (1 << k)))
1265                                 continue;
1266                         if (i + 1 > macvlan->vlan_num) {
1267                                 PMD_INIT_LOG(ERR, "vlan number not match");
1268                                 return;
1269                         }
1270                         fm10k_mbx_lock(hw);
1271                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1272                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1273                         fm10k_mbx_unlock(hw);
1274                         i++;
1275                 }
1276         }
1277 }
1278
1279 /* Add/Remove a MAC address, and update filters to VMDQ */
1280 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1281                 const u8 *mac, bool add, uint32_t pool)
1282 {
1283         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1284         struct fm10k_macvlan_filter_info *macvlan;
1285         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1286         uint32_t i;
1287
1288         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1289         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1290
1291         if (pool > macvlan->nb_queue_pools) {
1292                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1293                         " Max pool is %u",
1294                         pool, macvlan->nb_queue_pools);
1295                 return;
1296         }
1297         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1298                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1299                         continue;
1300                 fm10k_mbx_lock(hw);
1301                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1302                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1303                 fm10k_mbx_unlock(hw);
1304         }
1305 }
1306
1307 /* Add/Remove a MAC address, and update filters */
1308 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1309                 const u8 *mac, bool add, uint32_t pool)
1310 {
1311         struct fm10k_macvlan_filter_info *macvlan;
1312
1313         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1314
1315         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1316                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1317         else
1318                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1319
1320         if (add)
1321                 macvlan->mac_num++;
1322         else
1323                 macvlan->mac_num--;
1324 }
1325
1326 /* Add a MAC address, and update filters */
1327 static void
1328 fm10k_macaddr_add(struct rte_eth_dev *dev,
1329                 struct ether_addr *mac_addr,
1330                 uint32_t index,
1331                 uint32_t pool)
1332 {
1333         struct fm10k_macvlan_filter_info *macvlan;
1334
1335         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1336         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1337         macvlan->mac_vmdq_id[index] = pool;
1338 }
1339
1340 /* Remove a MAC address, and update filters */
1341 static void
1342 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1343 {
1344         struct rte_eth_dev_data *data = dev->data;
1345         struct fm10k_macvlan_filter_info *macvlan;
1346
1347         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1348         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1349                         FALSE, macvlan->mac_vmdq_id[index]);
1350         macvlan->mac_vmdq_id[index] = 0;
1351 }
1352
1353 static inline int
1354 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1355 {
1356         if ((request < min) || (request > max) || ((request % mult) != 0))
1357                 return -1;
1358         else
1359                 return 0;
1360 }
1361
1362 /*
1363  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1364  * the physical address is required. If the memzone is already created, then
1365  * this function returns a pointer to the existing memzone.
1366  */
1367 static inline const struct rte_memzone *
1368 allocate_hw_ring(const char *driver_name, const char *ring_name,
1369         uint8_t port_id, uint16_t queue_id, int socket_id,
1370         uint32_t size, uint32_t align)
1371 {
1372         char name[RTE_MEMZONE_NAMESIZE];
1373         const struct rte_memzone *mz;
1374
1375         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1376                  driver_name, ring_name, port_id, queue_id, socket_id);
1377
1378         /* return the memzone if it already exists */
1379         mz = rte_memzone_lookup(name);
1380         if (mz)
1381                 return mz;
1382
1383 #ifdef RTE_LIBRTE_XEN_DOM0
1384         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1385                                            RTE_PGSIZE_2M);
1386 #else
1387         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1388 #endif
1389 }
1390
1391 static inline int
1392 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1393 {
1394         if ((request < min) || (request > max) || ((div % request) != 0))
1395                 return -1;
1396         else
1397                 return 0;
1398 }
1399
1400 static inline int
1401 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1402 {
1403         uint16_t rx_free_thresh;
1404
1405         if (conf->rx_free_thresh == 0)
1406                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1407         else
1408                 rx_free_thresh = conf->rx_free_thresh;
1409
1410         /* make sure the requested threshold satisfies the constraints */
1411         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1412                         FM10K_RX_FREE_THRESH_MAX(q),
1413                         FM10K_RX_FREE_THRESH_DIV(q),
1414                         rx_free_thresh)) {
1415                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1416                         "less than or equal to %u, "
1417                         "greater than or equal to %u, "
1418                         "and a divisor of %u",
1419                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1420                         FM10K_RX_FREE_THRESH_MIN(q),
1421                         FM10K_RX_FREE_THRESH_DIV(q));
1422                 return (-EINVAL);
1423         }
1424
1425         q->alloc_thresh = rx_free_thresh;
1426         q->drop_en = conf->rx_drop_en;
1427         q->rx_deferred_start = conf->rx_deferred_start;
1428
1429         return 0;
1430 }
1431
1432 /*
1433  * Hardware requires specific alignment for Rx packet buffers. At
1434  * least one of the following two conditions must be satisfied.
1435  *  1. Address is 512B aligned
1436  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1437  *
1438  * As such, the driver may need to adjust the DMA address within the
1439  * buffer by up to 512B.
1440  *
1441  * return 1 if the element size is valid, otherwise return 0.
1442  */
1443 static int
1444 mempool_element_size_valid(struct rte_mempool *mp)
1445 {
1446         uint32_t min_size;
1447
1448         /* elt_size includes mbuf header and headroom */
1449         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1450                         RTE_PKTMBUF_HEADROOM;
1451
1452         /* account for up to 512B of alignment */
1453         min_size -= FM10K_RX_DATABUF_ALIGN;
1454
1455         /* sanity check for overflow */
1456         if (min_size > mp->elt_size)
1457                 return 0;
1458
1459         /* size is valid */
1460         return 1;
1461 }
1462
1463 static int
1464 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1465         uint16_t nb_desc, unsigned int socket_id,
1466         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1467 {
1468         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1469         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1470         struct fm10k_rx_queue *q;
1471         const struct rte_memzone *mz;
1472
1473         PMD_INIT_FUNC_TRACE();
1474
1475         /* make sure the mempool element size can account for alignment. */
1476         if (!mempool_element_size_valid(mp)) {
1477                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1478                 return (-EINVAL);
1479         }
1480
1481         /* make sure a valid number of descriptors have been requested */
1482         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1483                                 FM10K_MULT_RX_DESC, nb_desc)) {
1484                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1485                         "less than or equal to %"PRIu32", "
1486                         "greater than or equal to %u, "
1487                         "and a multiple of %u",
1488                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1489                         FM10K_MULT_RX_DESC);
1490                 return (-EINVAL);
1491         }
1492
1493         /*
1494          * if this queue existed already, free the associated memory. The
1495          * queue cannot be reused in case we need to allocate memory on
1496          * different socket than was previously used.
1497          */
1498         if (dev->data->rx_queues[queue_id] != NULL) {
1499                 rx_queue_free(dev->data->rx_queues[queue_id]);
1500                 dev->data->rx_queues[queue_id] = NULL;
1501         }
1502
1503         /* allocate memory for the queue structure */
1504         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1505                                 socket_id);
1506         if (q == NULL) {
1507                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1508                 return (-ENOMEM);
1509         }
1510
1511         /* setup queue */
1512         q->mp = mp;
1513         q->nb_desc = nb_desc;
1514         q->port_id = dev->data->port_id;
1515         q->queue_id = queue_id;
1516         q->tail_ptr = (volatile uint32_t *)
1517                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1518         if (handle_rxconf(q, conf))
1519                 return (-EINVAL);
1520
1521         /* allocate memory for the software ring */
1522         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1523                                         nb_desc * sizeof(struct rte_mbuf *),
1524                                         RTE_CACHE_LINE_SIZE, socket_id);
1525         if (q->sw_ring == NULL) {
1526                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1527                 rte_free(q);
1528                 return (-ENOMEM);
1529         }
1530
1531         /*
1532          * allocate memory for the hardware descriptor ring. A memzone large
1533          * enough to hold the maximum ring size is requested to allow for
1534          * resizing in later calls to the queue setup function.
1535          */
1536         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1537                                 dev->data->port_id, queue_id, socket_id,
1538                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1539         if (mz == NULL) {
1540                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1541                 rte_free(q->sw_ring);
1542                 rte_free(q);
1543                 return (-ENOMEM);
1544         }
1545         q->hw_ring = mz->addr;
1546 #ifdef RTE_LIBRTE_XEN_DOM0
1547         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1548 #else
1549         q->hw_ring_phys_addr = mz->phys_addr;
1550 #endif
1551
1552         /* Check if number of descs satisfied Vector requirement */
1553         if (!rte_is_power_of_2(nb_desc)) {
1554                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1555                                     "preconditions - canceling the feature for "
1556                                     "the whole port[%d]",
1557                              q->queue_id, q->port_id);
1558                 dev_info->rx_vec_allowed = false;
1559         } else
1560                 fm10k_rxq_vec_setup(q);
1561
1562         dev->data->rx_queues[queue_id] = q;
1563         return 0;
1564 }
1565
1566 static void
1567 fm10k_rx_queue_release(void *queue)
1568 {
1569         PMD_INIT_FUNC_TRACE();
1570
1571         rx_queue_free(queue);
1572 }
1573
1574 static inline int
1575 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1576 {
1577         uint16_t tx_free_thresh;
1578         uint16_t tx_rs_thresh;
1579
1580         /* constraint MACROs require that tx_free_thresh is configured
1581          * before tx_rs_thresh */
1582         if (conf->tx_free_thresh == 0)
1583                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1584         else
1585                 tx_free_thresh = conf->tx_free_thresh;
1586
1587         /* make sure the requested threshold satisfies the constraints */
1588         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1589                         FM10K_TX_FREE_THRESH_MAX(q),
1590                         FM10K_TX_FREE_THRESH_DIV(q),
1591                         tx_free_thresh)) {
1592                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1593                         "less than or equal to %u, "
1594                         "greater than or equal to %u, "
1595                         "and a divisor of %u",
1596                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1597                         FM10K_TX_FREE_THRESH_MIN(q),
1598                         FM10K_TX_FREE_THRESH_DIV(q));
1599                 return (-EINVAL);
1600         }
1601
1602         q->free_thresh = tx_free_thresh;
1603
1604         if (conf->tx_rs_thresh == 0)
1605                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1606         else
1607                 tx_rs_thresh = conf->tx_rs_thresh;
1608
1609         q->tx_deferred_start = conf->tx_deferred_start;
1610
1611         /* make sure the requested threshold satisfies the constraints */
1612         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1613                         FM10K_TX_RS_THRESH_MAX(q),
1614                         FM10K_TX_RS_THRESH_DIV(q),
1615                         tx_rs_thresh)) {
1616                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1617                         "less than or equal to %u, "
1618                         "greater than or equal to %u, "
1619                         "and a divisor of %u",
1620                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1621                         FM10K_TX_RS_THRESH_MIN(q),
1622                         FM10K_TX_RS_THRESH_DIV(q));
1623                 return (-EINVAL);
1624         }
1625
1626         q->rs_thresh = tx_rs_thresh;
1627
1628         return 0;
1629 }
1630
1631 static int
1632 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1633         uint16_t nb_desc, unsigned int socket_id,
1634         const struct rte_eth_txconf *conf)
1635 {
1636         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1637         struct fm10k_tx_queue *q;
1638         const struct rte_memzone *mz;
1639
1640         PMD_INIT_FUNC_TRACE();
1641
1642         /* make sure a valid number of descriptors have been requested */
1643         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1644                                 FM10K_MULT_TX_DESC, nb_desc)) {
1645                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1646                         "less than or equal to %"PRIu32", "
1647                         "greater than or equal to %u, "
1648                         "and a multiple of %u",
1649                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1650                         FM10K_MULT_TX_DESC);
1651                 return (-EINVAL);
1652         }
1653
1654         /*
1655          * if this queue existed already, free the associated memory. The
1656          * queue cannot be reused in case we need to allocate memory on
1657          * different socket than was previously used.
1658          */
1659         if (dev->data->tx_queues[queue_id] != NULL) {
1660                 tx_queue_free(dev->data->tx_queues[queue_id]);
1661                 dev->data->tx_queues[queue_id] = NULL;
1662         }
1663
1664         /* allocate memory for the queue structure */
1665         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1666                                 socket_id);
1667         if (q == NULL) {
1668                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1669                 return (-ENOMEM);
1670         }
1671
1672         /* setup queue */
1673         q->nb_desc = nb_desc;
1674         q->port_id = dev->data->port_id;
1675         q->queue_id = queue_id;
1676         q->tail_ptr = (volatile uint32_t *)
1677                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1678         if (handle_txconf(q, conf))
1679                 return (-EINVAL);
1680
1681         /* allocate memory for the software ring */
1682         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1683                                         nb_desc * sizeof(struct rte_mbuf *),
1684                                         RTE_CACHE_LINE_SIZE, socket_id);
1685         if (q->sw_ring == NULL) {
1686                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1687                 rte_free(q);
1688                 return (-ENOMEM);
1689         }
1690
1691         /*
1692          * allocate memory for the hardware descriptor ring. A memzone large
1693          * enough to hold the maximum ring size is requested to allow for
1694          * resizing in later calls to the queue setup function.
1695          */
1696         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1697                                 dev->data->port_id, queue_id, socket_id,
1698                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1699         if (mz == NULL) {
1700                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1701                 rte_free(q->sw_ring);
1702                 rte_free(q);
1703                 return (-ENOMEM);
1704         }
1705         q->hw_ring = mz->addr;
1706 #ifdef RTE_LIBRTE_XEN_DOM0
1707         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1708 #else
1709         q->hw_ring_phys_addr = mz->phys_addr;
1710 #endif
1711
1712         /*
1713          * allocate memory for the RS bit tracker. Enough slots to hold the
1714          * descriptor index for each RS bit needing to be set are required.
1715          */
1716         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1717                                 ((nb_desc + 1) / q->rs_thresh) *
1718                                 sizeof(uint16_t),
1719                                 RTE_CACHE_LINE_SIZE, socket_id);
1720         if (q->rs_tracker.list == NULL) {
1721                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1722                 rte_free(q->sw_ring);
1723                 rte_free(q);
1724                 return (-ENOMEM);
1725         }
1726
1727         dev->data->tx_queues[queue_id] = q;
1728         return 0;
1729 }
1730
1731 static void
1732 fm10k_tx_queue_release(void *queue)
1733 {
1734         PMD_INIT_FUNC_TRACE();
1735
1736         tx_queue_free(queue);
1737 }
1738
1739 static int
1740 fm10k_reta_update(struct rte_eth_dev *dev,
1741                         struct rte_eth_rss_reta_entry64 *reta_conf,
1742                         uint16_t reta_size)
1743 {
1744         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1745         uint16_t i, j, idx, shift;
1746         uint8_t mask;
1747         uint32_t reta;
1748
1749         PMD_INIT_FUNC_TRACE();
1750
1751         if (reta_size > FM10K_MAX_RSS_INDICES) {
1752                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1753                         "(%d) doesn't match the number hardware can supported "
1754                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1755                 return -EINVAL;
1756         }
1757
1758         /*
1759          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1760          * 128-entries in 32 registers
1761          */
1762         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1763                 idx = i / RTE_RETA_GROUP_SIZE;
1764                 shift = i % RTE_RETA_GROUP_SIZE;
1765                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1766                                 BIT_MASK_PER_UINT32);
1767                 if (mask == 0)
1768                         continue;
1769
1770                 reta = 0;
1771                 if (mask != BIT_MASK_PER_UINT32)
1772                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1773
1774                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1775                         if (mask & (0x1 << j)) {
1776                                 if (mask != 0xF)
1777                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1778                                 reta |= reta_conf[idx].reta[shift + j] <<
1779                                                 (CHAR_BIT * j);
1780                         }
1781                 }
1782                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1783         }
1784
1785         return 0;
1786 }
1787
1788 static int
1789 fm10k_reta_query(struct rte_eth_dev *dev,
1790                         struct rte_eth_rss_reta_entry64 *reta_conf,
1791                         uint16_t reta_size)
1792 {
1793         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1794         uint16_t i, j, idx, shift;
1795         uint8_t mask;
1796         uint32_t reta;
1797
1798         PMD_INIT_FUNC_TRACE();
1799
1800         if (reta_size < FM10K_MAX_RSS_INDICES) {
1801                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1802                         "(%d) doesn't match the number hardware can supported "
1803                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1804                 return -EINVAL;
1805         }
1806
1807         /*
1808          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1809          * 128-entries in 32 registers
1810          */
1811         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1812                 idx = i / RTE_RETA_GROUP_SIZE;
1813                 shift = i % RTE_RETA_GROUP_SIZE;
1814                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1815                                 BIT_MASK_PER_UINT32);
1816                 if (mask == 0)
1817                         continue;
1818
1819                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1820                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1821                         if (mask & (0x1 << j))
1822                                 reta_conf[idx].reta[shift + j] = ((reta >>
1823                                         CHAR_BIT * j) & UINT8_MAX);
1824                 }
1825         }
1826
1827         return 0;
1828 }
1829
1830 static int
1831 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1832         struct rte_eth_rss_conf *rss_conf)
1833 {
1834         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1835         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1836         uint32_t mrqc;
1837         uint64_t hf = rss_conf->rss_hf;
1838         int i;
1839
1840         PMD_INIT_FUNC_TRACE();
1841
1842         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1843                 FM10K_RSSRK_ENTRIES_PER_REG)
1844                 return -EINVAL;
1845
1846         if (hf == 0)
1847                 return -EINVAL;
1848
1849         mrqc = 0;
1850         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1851         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1852         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1853         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1854         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1855         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1856         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1857         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1858         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1859
1860         /* If the mapping doesn't fit any supported, return */
1861         if (mrqc == 0)
1862                 return -EINVAL;
1863
1864         if (key != NULL)
1865                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1866                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1867
1868         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1869
1870         return 0;
1871 }
1872
1873 static int
1874 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1875         struct rte_eth_rss_conf *rss_conf)
1876 {
1877         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1878         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1879         uint32_t mrqc;
1880         uint64_t hf;
1881         int i;
1882
1883         PMD_INIT_FUNC_TRACE();
1884
1885         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1886                                 FM10K_RSSRK_ENTRIES_PER_REG)
1887                 return -EINVAL;
1888
1889         if (key != NULL)
1890                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1891                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1892
1893         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1894         hf = 0;
1895         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1896         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1897         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1898         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1899         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1900         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1901         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1902         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1903         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1904
1905         rss_conf->rss_hf = hf;
1906
1907         return 0;
1908 }
1909
1910 static void
1911 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1912 {
1913         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1914         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1915
1916         /* Bind all local non-queue interrupt to vector 0 */
1917         int_map |= 0;
1918
1919         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1920         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1921         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1922         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1923         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1924         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1925
1926         /* Enable misc causes */
1927         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1928                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1929                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1930                                 FM10K_EIMR_ENABLE(MAILBOX) |
1931                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1932                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1933                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1934                                 FM10K_EIMR_ENABLE(VFLR));
1935
1936         /* Enable ITR 0 */
1937         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1938                                         FM10K_ITR_MASK_CLEAR);
1939         FM10K_WRITE_FLUSH(hw);
1940 }
1941
1942 static void
1943 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1944 {
1945         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1946         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1947
1948         int_map |= 0;
1949
1950         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1951         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1952         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1953         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1954         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1955         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1956
1957         /* Disable misc causes */
1958         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1959                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1960                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1961                                 FM10K_EIMR_DISABLE(MAILBOX) |
1962                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1963                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1964                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1965                                 FM10K_EIMR_DISABLE(VFLR));
1966
1967         /* Disable ITR 0 */
1968         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1969         FM10K_WRITE_FLUSH(hw);
1970 }
1971
1972 static void
1973 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1974 {
1975         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1976         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1977
1978         /* Bind all local non-queue interrupt to vector 0 */
1979         int_map |= 0;
1980
1981         /* Only INT 0 available, other 15 are reserved. */
1982         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1983
1984         /* Enable ITR 0 */
1985         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1986                                         FM10K_ITR_MASK_CLEAR);
1987         FM10K_WRITE_FLUSH(hw);
1988 }
1989
1990 static void
1991 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1992 {
1993         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1994         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1995
1996         int_map |= 0;
1997
1998         /* Only INT 0 available, other 15 are reserved. */
1999         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2000
2001         /* Disable ITR 0 */
2002         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2003         FM10K_WRITE_FLUSH(hw);
2004 }
2005
2006 static int
2007 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2008 {
2009         struct fm10k_fault fault;
2010         int err;
2011         const char *estr = "Unknown error";
2012
2013         /* Process PCA fault */
2014         if (eicr & FM10K_EICR_PCA_FAULT) {
2015                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2016                 if (err)
2017                         goto error;
2018                 switch (fault.type) {
2019                 case PCA_NO_FAULT:
2020                         estr = "PCA_NO_FAULT"; break;
2021                 case PCA_UNMAPPED_ADDR:
2022                         estr = "PCA_UNMAPPED_ADDR"; break;
2023                 case PCA_BAD_QACCESS_PF:
2024                         estr = "PCA_BAD_QACCESS_PF"; break;
2025                 case PCA_BAD_QACCESS_VF:
2026                         estr = "PCA_BAD_QACCESS_VF"; break;
2027                 case PCA_MALICIOUS_REQ:
2028                         estr = "PCA_MALICIOUS_REQ"; break;
2029                 case PCA_POISONED_TLP:
2030                         estr = "PCA_POISONED_TLP"; break;
2031                 case PCA_TLP_ABORT:
2032                         estr = "PCA_TLP_ABORT"; break;
2033                 default:
2034                         goto error;
2035                 }
2036                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2037                         estr, fault.func ? "VF" : "PF", fault.func,
2038                         fault.address, fault.specinfo);
2039         }
2040
2041         /* Process THI fault */
2042         if (eicr & FM10K_EICR_THI_FAULT) {
2043                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2044                 if (err)
2045                         goto error;
2046                 switch (fault.type) {
2047                 case THI_NO_FAULT:
2048                         estr = "THI_NO_FAULT"; break;
2049                 case THI_MAL_DIS_Q_FAULT:
2050                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2051                 default:
2052                         goto error;
2053                 }
2054                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2055                         estr, fault.func ? "VF" : "PF", fault.func,
2056                         fault.address, fault.specinfo);
2057         }
2058
2059         /* Process FUM fault */
2060         if (eicr & FM10K_EICR_FUM_FAULT) {
2061                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2062                 if (err)
2063                         goto error;
2064                 switch (fault.type) {
2065                 case FUM_NO_FAULT:
2066                         estr = "FUM_NO_FAULT"; break;
2067                 case FUM_UNMAPPED_ADDR:
2068                         estr = "FUM_UNMAPPED_ADDR"; break;
2069                 case FUM_POISONED_TLP:
2070                         estr = "FUM_POISONED_TLP"; break;
2071                 case FUM_BAD_VF_QACCESS:
2072                         estr = "FUM_BAD_VF_QACCESS"; break;
2073                 case FUM_ADD_DECODE_ERR:
2074                         estr = "FUM_ADD_DECODE_ERR"; break;
2075                 case FUM_RO_ERROR:
2076                         estr = "FUM_RO_ERROR"; break;
2077                 case FUM_QPRC_CRC_ERROR:
2078                         estr = "FUM_QPRC_CRC_ERROR"; break;
2079                 case FUM_CSR_TIMEOUT:
2080                         estr = "FUM_CSR_TIMEOUT"; break;
2081                 case FUM_INVALID_TYPE:
2082                         estr = "FUM_INVALID_TYPE"; break;
2083                 case FUM_INVALID_LENGTH:
2084                         estr = "FUM_INVALID_LENGTH"; break;
2085                 case FUM_INVALID_BE:
2086                         estr = "FUM_INVALID_BE"; break;
2087                 case FUM_INVALID_ALIGN:
2088                         estr = "FUM_INVALID_ALIGN"; break;
2089                 default:
2090                         goto error;
2091                 }
2092                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2093                         estr, fault.func ? "VF" : "PF", fault.func,
2094                         fault.address, fault.specinfo);
2095         }
2096
2097         return 0;
2098 error:
2099         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2100         return err;
2101 }
2102
2103 /**
2104  * PF interrupt handler triggered by NIC for handling specific interrupt.
2105  *
2106  * @param handle
2107  *  Pointer to interrupt handle.
2108  * @param param
2109  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2110  *
2111  * @return
2112  *  void
2113  */
2114 static void
2115 fm10k_dev_interrupt_handler_pf(
2116                         __rte_unused struct rte_intr_handle *handle,
2117                         void *param)
2118 {
2119         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2120         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2121         uint32_t cause, status;
2122
2123         if (hw->mac.type != fm10k_mac_pf)
2124                 return;
2125
2126         cause = FM10K_READ_REG(hw, FM10K_EICR);
2127
2128         /* Handle PCI fault cases */
2129         if (cause & FM10K_EICR_FAULT_MASK) {
2130                 PMD_INIT_LOG(ERR, "INT: find fault!");
2131                 fm10k_dev_handle_fault(hw, cause);
2132         }
2133
2134         /* Handle switch up/down */
2135         if (cause & FM10K_EICR_SWITCHNOTREADY)
2136                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2137
2138         if (cause & FM10K_EICR_SWITCHREADY)
2139                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2140
2141         /* Handle mailbox message */
2142         fm10k_mbx_lock(hw);
2143         hw->mbx.ops.process(hw, &hw->mbx);
2144         fm10k_mbx_unlock(hw);
2145
2146         /* Handle SRAM error */
2147         if (cause & FM10K_EICR_SRAMERROR) {
2148                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2149
2150                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2151                 /* Write to clear pending bits */
2152                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2153
2154                 /* Todo: print out error message after shared code  updates */
2155         }
2156
2157         /* Clear these 3 events if having any */
2158         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2159                  FM10K_EICR_SWITCHREADY;
2160         if (cause)
2161                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2162
2163         /* Re-enable interrupt from device side */
2164         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2165                                         FM10K_ITR_MASK_CLEAR);
2166         /* Re-enable interrupt from host side */
2167         rte_intr_enable(&(dev->pci_dev->intr_handle));
2168 }
2169
2170 /**
2171  * VF interrupt handler triggered by NIC for handling specific interrupt.
2172  *
2173  * @param handle
2174  *  Pointer to interrupt handle.
2175  * @param param
2176  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2177  *
2178  * @return
2179  *  void
2180  */
2181 static void
2182 fm10k_dev_interrupt_handler_vf(
2183                         __rte_unused struct rte_intr_handle *handle,
2184                         void *param)
2185 {
2186         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2187         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2188
2189         if (hw->mac.type != fm10k_mac_vf)
2190                 return;
2191
2192         /* Handle mailbox message if lock is acquired */
2193         fm10k_mbx_lock(hw);
2194         hw->mbx.ops.process(hw, &hw->mbx);
2195         fm10k_mbx_unlock(hw);
2196
2197         /* Re-enable interrupt from device side */
2198         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2199                                         FM10K_ITR_MASK_CLEAR);
2200         /* Re-enable interrupt from host side */
2201         rte_intr_enable(&(dev->pci_dev->intr_handle));
2202 }
2203
2204 /* Mailbox message handler in VF */
2205 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2206         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2207         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2208         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2209         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2210 };
2211
2212 /* Mailbox message handler in PF */
2213 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2214         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2215         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2216         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2217         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2218         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2219         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2220         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2221 };
2222
2223 static int
2224 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2225 {
2226         int err;
2227
2228         /* Initialize mailbox lock */
2229         fm10k_mbx_initlock(hw);
2230
2231         /* Replace default message handler with new ones */
2232         if (hw->mac.type == fm10k_mac_pf)
2233                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2234         else
2235                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2236
2237         if (err) {
2238                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2239                                 err);
2240                 return err;
2241         }
2242         /* Connect to SM for PF device or PF for VF device */
2243         return hw->mbx.ops.connect(hw, &hw->mbx);
2244 }
2245
2246 static void
2247 fm10k_close_mbx_service(struct fm10k_hw *hw)
2248 {
2249         /* Disconnect from SM for PF device or PF for VF device */
2250         hw->mbx.ops.disconnect(hw, &hw->mbx);
2251 }
2252
2253 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2254         .dev_configure          = fm10k_dev_configure,
2255         .dev_start              = fm10k_dev_start,
2256         .dev_stop               = fm10k_dev_stop,
2257         .dev_close              = fm10k_dev_close,
2258         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2259         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2260         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2261         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2262         .stats_get              = fm10k_stats_get,
2263         .stats_reset            = fm10k_stats_reset,
2264         .link_update            = fm10k_link_update,
2265         .dev_infos_get          = fm10k_dev_infos_get,
2266         .vlan_filter_set        = fm10k_vlan_filter_set,
2267         .vlan_offload_set       = fm10k_vlan_offload_set,
2268         .mac_addr_add           = fm10k_macaddr_add,
2269         .mac_addr_remove        = fm10k_macaddr_remove,
2270         .rx_queue_start         = fm10k_dev_rx_queue_start,
2271         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2272         .tx_queue_start         = fm10k_dev_tx_queue_start,
2273         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2274         .rx_queue_setup         = fm10k_rx_queue_setup,
2275         .rx_queue_release       = fm10k_rx_queue_release,
2276         .tx_queue_setup         = fm10k_tx_queue_setup,
2277         .tx_queue_release       = fm10k_tx_queue_release,
2278         .reta_update            = fm10k_reta_update,
2279         .reta_query             = fm10k_reta_query,
2280         .rss_hash_update        = fm10k_rss_hash_update,
2281         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2282 };
2283
2284 static void
2285 fm10k_params_init(struct rte_eth_dev *dev)
2286 {
2287         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2288         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2289
2290         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2291          * there is no way to get link status without reading BAR4.  Until this
2292          * works, assume we have maximum bandwidth.
2293          * @todo - fix bus info
2294          */
2295         hw->bus_caps.speed = fm10k_bus_speed_8000;
2296         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2297         hw->bus_caps.payload = fm10k_bus_payload_512;
2298         hw->bus.speed = fm10k_bus_speed_8000;
2299         hw->bus.width = fm10k_bus_width_pcie_x8;
2300         hw->bus.payload = fm10k_bus_payload_256;
2301
2302         info->rx_vec_allowed = true;
2303 }
2304
2305 static int
2306 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2307 {
2308         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2309         int diag;
2310         struct fm10k_macvlan_filter_info *macvlan;
2311
2312         PMD_INIT_FUNC_TRACE();
2313
2314         dev->dev_ops = &fm10k_eth_dev_ops;
2315         dev->rx_pkt_burst = &fm10k_recv_pkts;
2316         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2317
2318         if (dev->data->scattered_rx)
2319                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2320
2321         /* only initialize in the primary process */
2322         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2323                 return 0;
2324
2325         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2326         memset(macvlan, 0, sizeof(*macvlan));
2327         /* Vendor and Device ID need to be set before init of shared code */
2328         memset(hw, 0, sizeof(*hw));
2329         hw->device_id = dev->pci_dev->id.device_id;
2330         hw->vendor_id = dev->pci_dev->id.vendor_id;
2331         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2332         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2333         hw->revision_id = 0;
2334         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2335         if (hw->hw_addr == NULL) {
2336                 PMD_INIT_LOG(ERR, "Bad mem resource."
2337                         " Try to blacklist unused devices.");
2338                 return -EIO;
2339         }
2340
2341         /* Store fm10k_adapter pointer */
2342         hw->back = dev->data->dev_private;
2343
2344         /* Initialize the shared code */
2345         diag = fm10k_init_shared_code(hw);
2346         if (diag != FM10K_SUCCESS) {
2347                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2348                 return -EIO;
2349         }
2350
2351         /* Initialize parameters */
2352         fm10k_params_init(dev);
2353
2354         /* Initialize the hw */
2355         diag = fm10k_init_hw(hw);
2356         if (diag != FM10K_SUCCESS) {
2357                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2358                 return -EIO;
2359         }
2360
2361         /* Initialize MAC address(es) */
2362         dev->data->mac_addrs = rte_zmalloc("fm10k",
2363                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2364         if (dev->data->mac_addrs == NULL) {
2365                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2366                 return -ENOMEM;
2367         }
2368
2369         diag = fm10k_read_mac_addr(hw);
2370
2371         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2372                         &dev->data->mac_addrs[0]);
2373
2374         if (diag != FM10K_SUCCESS ||
2375                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2376
2377                 /* Generate a random addr */
2378                 eth_random_addr(hw->mac.addr);
2379                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2380                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2381                 &dev->data->mac_addrs[0]);
2382         }
2383
2384         /* Reset the hw statistics */
2385         fm10k_stats_reset(dev);
2386
2387         /* Reset the hw */
2388         diag = fm10k_reset_hw(hw);
2389         if (diag != FM10K_SUCCESS) {
2390                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2391                 return -EIO;
2392         }
2393
2394         /* Setup mailbox service */
2395         diag = fm10k_setup_mbx_service(hw);
2396         if (diag != FM10K_SUCCESS) {
2397                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2398                 return -EIO;
2399         }
2400
2401         /*PF/VF has different interrupt handling mechanism */
2402         if (hw->mac.type == fm10k_mac_pf) {
2403                 /* register callback func to eal lib */
2404                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2405                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2406
2407                 /* enable MISC interrupt */
2408                 fm10k_dev_enable_intr_pf(dev);
2409         } else { /* VF */
2410                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2411                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2412
2413                 fm10k_dev_enable_intr_vf(dev);
2414         }
2415
2416         /* Enable uio intr after callback registered */
2417         rte_intr_enable(&(dev->pci_dev->intr_handle));
2418
2419         hw->mac.ops.update_int_moderator(hw);
2420
2421         /* Make sure Switch Manager is ready before going forward. */
2422         if (hw->mac.type == fm10k_mac_pf) {
2423                 int switch_ready = 0;
2424                 int i;
2425
2426                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2427                         fm10k_mbx_lock(hw);
2428                         hw->mac.ops.get_host_state(hw, &switch_ready);
2429                         fm10k_mbx_unlock(hw);
2430                         if (switch_ready)
2431                                 break;
2432                         /* Delay some time to acquire async LPORT_MAP info. */
2433                         rte_delay_us(WAIT_SWITCH_MSG_US);
2434                 }
2435
2436                 if (switch_ready == 0) {
2437                         PMD_INIT_LOG(ERR, "switch is not ready");
2438                         return -1;
2439                 }
2440         }
2441
2442         /*
2443          * Below function will trigger operations on mailbox, acquire lock to
2444          * avoid race condition from interrupt handler. Operations on mailbox
2445          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2446          * will handle and generate an interrupt to our side. Then,  FIFO in
2447          * mailbox will be touched.
2448          */
2449         fm10k_mbx_lock(hw);
2450         /* Enable port first */
2451         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2452
2453         /* Set unicast mode by default. App can change to other mode in other
2454          * API func.
2455          */
2456         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2457                                         FM10K_XCAST_MODE_NONE);
2458
2459         fm10k_mbx_unlock(hw);
2460
2461         /* Add default mac address */
2462         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2463                 MAIN_VSI_POOL_NUMBER);
2464
2465         return 0;
2466 }
2467
2468 static int
2469 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2470 {
2471         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2472
2473         PMD_INIT_FUNC_TRACE();
2474
2475         /* only uninitialize in the primary process */
2476         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2477                 return 0;
2478
2479         /* safe to close dev here */
2480         fm10k_dev_close(dev);
2481
2482         dev->dev_ops = NULL;
2483         dev->rx_pkt_burst = NULL;
2484         dev->tx_pkt_burst = NULL;
2485
2486         /* disable uio/vfio intr */
2487         rte_intr_disable(&(dev->pci_dev->intr_handle));
2488
2489         /*PF/VF has different interrupt handling mechanism */
2490         if (hw->mac.type == fm10k_mac_pf) {
2491                 /* disable interrupt */
2492                 fm10k_dev_disable_intr_pf(dev);
2493
2494                 /* unregister callback func to eal lib */
2495                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2496                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2497         } else {
2498                 /* disable interrupt */
2499                 fm10k_dev_disable_intr_vf(dev);
2500
2501                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2502                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2503         }
2504
2505         /* free mac memory */
2506         if (dev->data->mac_addrs) {
2507                 rte_free(dev->data->mac_addrs);
2508                 dev->data->mac_addrs = NULL;
2509         }
2510
2511         memset(hw, 0, sizeof(*hw));
2512
2513         return 0;
2514 }
2515
2516 /*
2517  * The set of PCI devices this driver supports. This driver will enable both PF
2518  * and SRIOV-VF devices.
2519  */
2520 static const struct rte_pci_id pci_id_fm10k_map[] = {
2521 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2522 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2523 #include "rte_pci_dev_ids.h"
2524         { .vendor_id = 0, /* sentinel */ },
2525 };
2526
2527 static struct eth_driver rte_pmd_fm10k = {
2528         .pci_drv = {
2529                 .name = "rte_pmd_fm10k",
2530                 .id_table = pci_id_fm10k_map,
2531                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2532         },
2533         .eth_dev_init = eth_fm10k_dev_init,
2534         .eth_dev_uninit = eth_fm10k_dev_uninit,
2535         .dev_private_size = sizeof(struct fm10k_adapter),
2536 };
2537
2538 /*
2539  * Driver initialization routine.
2540  * Invoked once at EAL init time.
2541  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2542  */
2543 static int
2544 rte_pmd_fm10k_init(__rte_unused const char *name,
2545         __rte_unused const char *params)
2546 {
2547         PMD_INIT_FUNC_TRACE();
2548         rte_eth_driver_register(&rte_pmd_fm10k);
2549         return 0;
2550 }
2551
2552 static struct rte_driver rte_fm10k_driver = {
2553         .type = PMD_PDEV,
2554         .init = rte_pmd_fm10k_init,
2555 };
2556
2557 PMD_REGISTER_DRIVER(rte_fm10k_driver);