drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  64 static int
  65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  67         const u8 *mac, bool add, uint32_t pool);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         return 0;
 125 }
 126
 127 /*
 128  * clean queue, descriptor rings, free software buffers used when stopping
 129  * device.
 130  */
 131 static inline void
 132 rx_queue_clean(struct fm10k_rx_queue *q)
 133 {
 134         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 135         uint32_t i;
 136         PMD_INIT_FUNC_TRACE();
 137
 138         /* zero descriptor rings */
 139         for (i = 0; i < q->nb_desc; ++i)
 140                 q->hw_ring[i] = zero;
 141
 142         /* free software buffers */
 143         for (i = 0; i < q->nb_desc; ++i) {
 144                 if (q->sw_ring[i]) {
 145                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 146                         q->sw_ring[i] = NULL;
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * free all queue memory used when releasing the queue (i.e. configure)
 153  */
 154 static inline void
 155 rx_queue_free(struct fm10k_rx_queue *q)
 156 {
 157         PMD_INIT_FUNC_TRACE();
 158         if (q) {
 159                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 160                 rx_queue_clean(q);
 161                 if (q->sw_ring) {
 162                         rte_free(q->sw_ring);
 163                         q->sw_ring = NULL;
 164                 }
 165                 rte_free(q);
 166                 q = NULL;
 167         }
 168 }
 169
 170 /*
 171  * disable RX queue, wait unitl HW finished necessary flush operation
 172  */
 173 static inline int
 174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 175 {
 176         uint32_t reg, i;
 177
 178         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 179         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 180                         reg & ~FM10K_RXQCTL_ENABLE);
 181
 182         /* Wait 100us at most */
 183         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 184                 rte_delay_us(1);
 185                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 186                 if (!(reg & FM10K_RXQCTL_ENABLE))
 187                         break;
 188         }
 189
 190         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 191                 return -1;
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * reset queue to initial state, allocate software buffers used when starting
 198  * device
 199  */
 200 static inline void
 201 tx_queue_reset(struct fm10k_tx_queue *q)
 202 {
 203         PMD_INIT_FUNC_TRACE();
 204         q->last_free = 0;
 205         q->next_free = 0;
 206         q->nb_used = 0;
 207         q->nb_free = q->nb_desc - 1;
 208         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 209         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 210 }
 211
 212 /*
 213  * clean queue, descriptor rings, free software buffers used when stopping
 214  * device
 215  */
 216 static inline void
 217 tx_queue_clean(struct fm10k_tx_queue *q)
 218 {
 219         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 220         uint32_t i;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         /* zero descriptor rings */
 224         for (i = 0; i < q->nb_desc; ++i)
 225                 q->hw_ring[i] = zero;
 226
 227         /* free software buffers */
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 if (q->sw_ring[i]) {
 230                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 231                         q->sw_ring[i] = NULL;
 232                 }
 233         }
 234 }
 235
 236 /*
 237  * free all queue memory used when releasing the queue (i.e. configure)
 238  */
 239 static inline void
 240 tx_queue_free(struct fm10k_tx_queue *q)
 241 {
 242         PMD_INIT_FUNC_TRACE();
 243         if (q) {
 244                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 245                 tx_queue_clean(q);
 246                 if (q->rs_tracker.list) {
 247                         rte_free(q->rs_tracker.list);
 248                         q->rs_tracker.list = NULL;
 249                 }
 250                 if (q->sw_ring) {
 251                         rte_free(q->sw_ring);
 252                         q->sw_ring = NULL;
 253                 }
 254                 rte_free(q);
 255                 q = NULL;
 256         }
 257 }
 258
 259 /*
 260  * disable TX queue, wait unitl HW finished necessary flush operation
 261  */
 262 static inline int
 263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 264 {
 265         uint32_t reg, i;
 266
 267         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 268         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 269                         reg & ~FM10K_TXDCTL_ENABLE);
 270
 271         /* Wait 100us at most */
 272         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 273                 rte_delay_us(1);
 274                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 275                 if (!(reg & FM10K_TXDCTL_ENABLE))
 276                         break;
 277         }
 278
 279         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 280                 return -1;
 281
 282         return 0;
 283 }
 284
 285 static int
 286 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 287 {
 288         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 291         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 292
 293         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 294
 295         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 296                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 297                 return -EINVAL;
 298         }
 299
 300         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 301                 return 0;
 302
 303         if (hw->mac.type == fm10k_mac_vf) {
 304                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 305                 return -EINVAL;
 306         }
 307
 308         /* Check VMDQ queue pool number */
 309         if (vmdq_conf->nb_queue_pools >
 310                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 311                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 312                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 313                         vmdq_conf->nb_queue_pools);
 314                 return -EINVAL;
 315         }
 316
 317         return 0;
 318 }
 319
 320 static int
 321 fm10k_dev_configure(struct rte_eth_dev *dev)
 322 {
 323         int ret;
 324
 325         PMD_INIT_FUNC_TRACE();
 326
 327         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 328                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 329         /* multipe queue mode checking */
 330         ret  = fm10k_check_mq_mode(dev);
 331         if (ret != 0) {
 332                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 333                             ret);
 334                 return ret;
 335         }
 336
 337         return 0;
 338 }
 339
 340 /* fls = find last set bit = 32 minus the number of leading zeros */
 341 #ifndef fls
 342 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 343 #endif
 344
 345 static void
 346 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 347 {
 348         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 349         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 350         uint32_t i;
 351
 352         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 353
 354         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 355                 if (!vmdq_conf->pool_map[i].pools)
 356                         continue;
 357                 fm10k_mbx_lock(hw);
 358                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 359                 fm10k_mbx_unlock(hw);
 360         }
 361 }
 362
 363 static void
 364 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 365 {
 366         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 367
 368         /* Add default mac address */
 369         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 370                 MAIN_VSI_POOL_NUMBER);
 371 }
 372
 373 static void
 374 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 375 {
 376         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 377         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 378         uint32_t mrqc, *key, i, reta, j;
 379         uint64_t hf;
 380
 381 #define RSS_KEY_SIZE 40
 382         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 383                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 384                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 385                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 386                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 387                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 388         };
 389
 390         if (dev->data->nb_rx_queues == 1 ||
 391             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 392             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 393                 return;
 394
 395         /* random key is rss_intel_key (default) or user provided (rss_key) */
 396         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 397                 key = (uint32_t *)rss_intel_key;
 398         else
 399                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 400
 401         /* Now fill our hash function seeds, 4 bytes at a time */
 402         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 403                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 404
 405         /*
 406          * Fill in redirection table
 407          * The byte-swap is needed because NIC registers are in
 408          * little-endian order.
 409          */
 410         reta = 0;
 411         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 412                 if (j == dev->data->nb_rx_queues)
 413                         j = 0;
 414                 reta = (reta << CHAR_BIT) | j;
 415                 if ((i & 3) == 3)
 416                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 417                                         rte_bswap32(reta));
 418         }
 419
 420         /*
 421          * Generate RSS hash based on packet types, TCP/UDP
 422          * port numbers and/or IPv4/v6 src and dst addresses
 423          */
 424         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 425         mrqc = 0;
 426         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 427         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 428         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 429         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 430         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 431         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 432         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 433         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 434         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 435
 436         if (mrqc == 0) {
 437                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 438                         "supported", hf);
 439                 return;
 440         }
 441
 442         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 443 }
 444
 445 static void
 446 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 447         uint16_t nb_lport_old, uint16_t nb_lport_new)
 448 {
 449         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 450         uint32_t i;
 451
 452         fm10k_mbx_lock(hw);
 453         /* Disable previous logic ports */
 454         if (nb_lport_old)
 455                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 456                         nb_lport_old, false);
 457         /* Enable new logic ports */
 458         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 459                 nb_lport_new, true);
 460         fm10k_mbx_unlock(hw);
 461
 462         for (i = 0; i < nb_lport_new; i++) {
 463                 /* Set unicast mode by default. App can change
 464                  * to other mode in other API func.
 465                  */
 466                 fm10k_mbx_lock(hw);
 467                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 468                         FM10K_XCAST_MODE_NONE);
 469                 fm10k_mbx_unlock(hw);
 470         }
 471 }
 472
 473 static void
 474 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 475 {
 476         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 477         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 478         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 479         struct fm10k_macvlan_filter_info *macvlan;
 480         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 481         uint16_t nb_lport_new, nb_lport_old;
 482
 483         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 484         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 485
 486         fm10k_dev_rss_configure(dev);
 487
 488         /* only PF supports VMDQ */
 489         if (hw->mac.type != fm10k_mac_pf)
 490                 return;
 491
 492         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 493                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 494
 495         /* no pool number change, no need to update logic port and VLAN/MAC */
 496         if (macvlan->nb_queue_pools == nb_queue_pools)
 497                 return;
 498
 499         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 500         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 501         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 502
 503         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 504         memset(dev->data->mac_addrs, 0,
 505                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 506         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 507                 &dev->data->mac_addrs[0]);
 508         memset(macvlan, 0, sizeof(*macvlan));
 509         macvlan->nb_queue_pools = nb_queue_pools;
 510
 511         if (nb_queue_pools)
 512                 fm10k_dev_vmdq_rx_configure(dev);
 513         else
 514                 fm10k_dev_pf_main_vsi_reset(dev);
 515 }
 516
 517 static int
 518 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 519 {
 520         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 521         int i, ret;
 522         struct fm10k_tx_queue *txq;
 523         uint64_t base_addr;
 524         uint32_t size;
 525
 526         /* Disable TXINT to avoid possible interrupt */
 527         for (i = 0; i < hw->mac.max_queues; i++)
 528                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 529                                 3 << FM10K_TXINT_TIMER_SHIFT);
 530
 531         /* Setup TX queue */
 532         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 533                 txq = dev->data->tx_queues[i];
 534                 base_addr = txq->hw_ring_phys_addr;
 535                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 536
 537                 /* disable queue to avoid issues while updating state */
 538                 ret = tx_queue_disable(hw, i);
 539                 if (ret) {
 540                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 541                         return -1;
 542                 }
 543
 544                 /* set location and size for descriptor ring */
 545                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 546                                 base_addr & UINT64_LOWER_32BITS_MASK);
 547                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 548                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 549                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 550         }
 551         return 0;
 552 }
 553
 554 static int
 555 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 556 {
 557         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 558         int i, ret;
 559         struct fm10k_rx_queue *rxq;
 560         uint64_t base_addr;
 561         uint32_t size;
 562         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 563         uint16_t buf_size;
 564
 565         /* Disable RXINT to avoid possible interrupt */
 566         for (i = 0; i < hw->mac.max_queues; i++)
 567                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 568                                 3 << FM10K_RXINT_TIMER_SHIFT);
 569
 570         /* Setup RX queues */
 571         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 572                 rxq = dev->data->rx_queues[i];
 573                 base_addr = rxq->hw_ring_phys_addr;
 574                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 575
 576                 /* disable queue to avoid issues while updating state */
 577                 ret = rx_queue_disable(hw, i);
 578                 if (ret) {
 579                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 580                         return -1;
 581                 }
 582
 583                 /* Setup the Base and Length of the Rx Descriptor Ring */
 584                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 585                                 base_addr & UINT64_LOWER_32BITS_MASK);
 586                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 587                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 588                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 589
 590                 /* Configure the Rx buffer size for one buff without split */
 591                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 592                         RTE_PKTMBUF_HEADROOM);
 593                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 594                  * reserved for this purpose, and the worst case could be 511B.
 595                  * But SRR reg assumes all buffers have the same size. In order
 596                  * to fill the gap, we'll have to consider the worst case and
 597                  * assume 512B is reserved. If we don't do so, it's possible
 598                  * for HW to overwrite data to next mbuf.
 599                  */
 600                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 601
 602                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 603                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 604
 605                 /* It adds dual VLAN length for supporting dual VLAN */
 606                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 607                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 608                         dev->data->dev_conf.rxmode.enable_scatter) {
 609                         uint32_t reg;
 610                         dev->data->scattered_rx = 1;
 611                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 612                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 613                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 614                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 615                 }
 616
 617                 /* Enable drop on empty, it's RO for VF */
 618                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 619                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 620
 621                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 622                 FM10K_WRITE_FLUSH(hw);
 623         }
 624
 625         /* Configure VMDQ/RSS if applicable */
 626         fm10k_dev_mq_rx_configure(dev);
 627         return 0;
 628 }
 629
 630 static int
 631 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 632 {
 633         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 634         int err = -1;
 635         uint32_t reg;
 636         struct fm10k_rx_queue *rxq;
 637
 638         PMD_INIT_FUNC_TRACE();
 639
 640         if (rx_queue_id < dev->data->nb_rx_queues) {
 641                 rxq = dev->data->rx_queues[rx_queue_id];
 642                 err = rx_queue_reset(rxq);
 643                 if (err == -ENOMEM) {
 644                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 645                         return err;
 646                 } else if (err == -EINVAL) {
 647                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 648                                 " %d", err);
 649                         return err;
 650                 }
 651
 652                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 653                  * Note: this must be done AFTER the queue is enabled on real
 654                  * hardware, but BEFORE the queue is enabled when using the
 655                  * emulation platform. Do it in both places for now and remove
 656                  * this comment and the following two register writes when the
 657                  * emulation platform is no longer being used.
 658                  */
 659                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 660                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 661
 662                 /* Set PF ownership flag for PF devices */
 663                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 664                 if (hw->mac.type == fm10k_mac_pf)
 665                         reg |= FM10K_RXQCTL_PF;
 666                 reg |= FM10K_RXQCTL_ENABLE;
 667                 /* enable RX queue */
 668                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 669                 FM10K_WRITE_FLUSH(hw);
 670
 671                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 672                  * Note: this must be done AFTER the queue is enabled
 673                  */
 674                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 675                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 676         }
 677
 678         return err;
 679 }
 680
 681 static int
 682 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 683 {
 684         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 685
 686         PMD_INIT_FUNC_TRACE();
 687
 688         if (rx_queue_id < dev->data->nb_rx_queues) {
 689                 /* Disable RX queue */
 690                 rx_queue_disable(hw, rx_queue_id);
 691
 692                 /* Free mbuf and clean HW ring */
 693                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 694         }
 695
 696         return 0;
 697 }
 698
 699 static int
 700 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 701 {
 702         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 703         /** @todo - this should be defined in the shared code */
 704 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 705         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 706         int err = 0;
 707
 708         PMD_INIT_FUNC_TRACE();
 709
 710         if (tx_queue_id < dev->data->nb_tx_queues) {
 711                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 712
 713                 /* reset head and tail pointers */
 714                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 715                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 716
 717                 /* enable TX queue */
 718                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 719                                         FM10K_TXDCTL_ENABLE | txdctl);
 720                 FM10K_WRITE_FLUSH(hw);
 721         } else
 722                 err = -1;
 723
 724         return err;
 725 }
 726
 727 static int
 728 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 729 {
 730         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 731
 732         PMD_INIT_FUNC_TRACE();
 733
 734         if (tx_queue_id < dev->data->nb_tx_queues) {
 735                 tx_queue_disable(hw, tx_queue_id);
 736                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 737         }
 738
 739         return 0;
 740 }
 741
 742 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 743 {
 744         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 745                 != FM10K_DGLORTMAP_NONE);
 746 }
 747
 748 static void
 749 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 750 {
 751         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 752         int status;
 753
 754         PMD_INIT_FUNC_TRACE();
 755
 756         /* Return if it didn't acquire valid glort range */
 757         if (!fm10k_glort_valid(hw))
 758                 return;
 759
 760         fm10k_mbx_lock(hw);
 761         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 762                                 FM10K_XCAST_MODE_PROMISC);
 763         fm10k_mbx_unlock(hw);
 764
 765         if (status != FM10K_SUCCESS)
 766                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 767 }
 768
 769 static void
 770 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 771 {
 772         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 773         uint8_t mode;
 774         int status;
 775
 776         PMD_INIT_FUNC_TRACE();
 777
 778         /* Return if it didn't acquire valid glort range */
 779         if (!fm10k_glort_valid(hw))
 780                 return;
 781
 782         if (dev->data->all_multicast == 1)
 783                 mode = FM10K_XCAST_MODE_ALLMULTI;
 784         else
 785                 mode = FM10K_XCAST_MODE_NONE;
 786
 787         fm10k_mbx_lock(hw);
 788         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 789                                 mode);
 790         fm10k_mbx_unlock(hw);
 791
 792         if (status != FM10K_SUCCESS)
 793                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 794 }
 795
 796 static void
 797 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 798 {
 799         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 800         int status;
 801
 802         PMD_INIT_FUNC_TRACE();
 803
 804         /* Return if it didn't acquire valid glort range */
 805         if (!fm10k_glort_valid(hw))
 806                 return;
 807
 808         /* If promiscuous mode is enabled, it doesn't make sense to enable
 809          * allmulticast and disable promiscuous since fm10k only can select
 810          * one of the modes.
 811          */
 812         if (dev->data->promiscuous) {
 813                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 814                         "needn't enable allmulticast");
 815                 return;
 816         }
 817
 818         fm10k_mbx_lock(hw);
 819         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 820                                 FM10K_XCAST_MODE_ALLMULTI);
 821         fm10k_mbx_unlock(hw);
 822
 823         if (status != FM10K_SUCCESS)
 824                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 825 }
 826
 827 static void
 828 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 829 {
 830         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 831         int status;
 832
 833         PMD_INIT_FUNC_TRACE();
 834
 835         /* Return if it didn't acquire valid glort range */
 836         if (!fm10k_glort_valid(hw))
 837                 return;
 838
 839         if (dev->data->promiscuous) {
 840                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 841                         "since promisc mode is enabled");
 842                 return;
 843         }
 844
 845         fm10k_mbx_lock(hw);
 846         /* Change mode to unicast mode */
 847         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 848                                 FM10K_XCAST_MODE_NONE);
 849         fm10k_mbx_unlock(hw);
 850
 851         if (status != FM10K_SUCCESS)
 852                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 853 }
 854
 855 static void
 856 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 857 {
 858         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859         uint32_t dglortdec, pool_len, rss_len, i;
 860         uint16_t nb_queue_pools;
 861         struct fm10k_macvlan_filter_info *macvlan;
 862
 863         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 864         nb_queue_pools = macvlan->nb_queue_pools;
 865         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 866         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 867         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 868
 869         /* Establish only MAP 0 as valid */
 870         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 871
 872         /* Configure VMDQ/RSS DGlort Decoder */
 873         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 874
 875         /* Invalidate all other GLORT entries */
 876         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 877                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 878                                 FM10K_DGLORTMAP_NONE);
 879 }
 880
 881 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 882 static int
 883 fm10k_dev_start(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886         int i, diag;
 887
 888         PMD_INIT_FUNC_TRACE();
 889
 890         /* stop, init, then start the hw */
 891         diag = fm10k_stop_hw(hw);
 892         if (diag != FM10K_SUCCESS) {
 893                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 894                 return -EIO;
 895         }
 896
 897         diag = fm10k_init_hw(hw);
 898         if (diag != FM10K_SUCCESS) {
 899                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 900                 return -EIO;
 901         }
 902
 903         diag = fm10k_start_hw(hw);
 904         if (diag != FM10K_SUCCESS) {
 905                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 906                 return -EIO;
 907         }
 908
 909         diag = fm10k_dev_tx_init(dev);
 910         if (diag) {
 911                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 912                 return diag;
 913         }
 914
 915         diag = fm10k_dev_rx_init(dev);
 916         if (diag) {
 917                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 918                 return diag;
 919         }
 920
 921         if (hw->mac.type == fm10k_mac_pf)
 922                 fm10k_dev_dglort_map_configure(dev);
 923
 924         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 925                 struct fm10k_rx_queue *rxq;
 926                 rxq = dev->data->rx_queues[i];
 927
 928                 if (rxq->rx_deferred_start)
 929                         continue;
 930                 diag = fm10k_dev_rx_queue_start(dev, i);
 931                 if (diag != 0) {
 932                         int j;
 933                         for (j = 0; j < i; ++j)
 934                                 rx_queue_clean(dev->data->rx_queues[j]);
 935                         return diag;
 936                 }
 937         }
 938
 939         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 940                 struct fm10k_tx_queue *txq;
 941                 txq = dev->data->tx_queues[i];
 942
 943                 if (txq->tx_deferred_start)
 944                         continue;
 945                 diag = fm10k_dev_tx_queue_start(dev, i);
 946                 if (diag != 0) {
 947                         int j;
 948                         for (j = 0; j < i; ++j)
 949                                 tx_queue_clean(dev->data->tx_queues[j]);
 950                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 951                                 rx_queue_clean(dev->data->rx_queues[j]);
 952                         return diag;
 953                 }
 954         }
 955
 956         /* Update default vlan when not in VMDQ mode */
 957         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 958                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 959
 960         return 0;
 961 }
 962
 963 static void
 964 fm10k_dev_stop(struct rte_eth_dev *dev)
 965 {
 966         int i;
 967
 968         PMD_INIT_FUNC_TRACE();
 969
 970         if (dev->data->tx_queues)
 971                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 972                         fm10k_dev_tx_queue_stop(dev, i);
 973
 974         if (dev->data->rx_queues)
 975                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 976                         fm10k_dev_rx_queue_stop(dev, i);
 977 }
 978
 979 static void
 980 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 981 {
 982         int i;
 983
 984         PMD_INIT_FUNC_TRACE();
 985
 986         if (dev->data->tx_queues) {
 987                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 988                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 989         }
 990
 991         if (dev->data->rx_queues) {
 992                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 993                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 994         }
 995 }
 996
 997 static void
 998 fm10k_dev_close(struct rte_eth_dev *dev)
 999 {
1000         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1001         uint16_t nb_lport;
1002         struct fm10k_macvlan_filter_info *macvlan;
1003
1004         PMD_INIT_FUNC_TRACE();
1005
1006         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1007         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1008         fm10k_mbx_lock(hw);
1009         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1010                 nb_lport, false);
1011         fm10k_mbx_unlock(hw);
1012
1013         /* Stop mailbox service first */
1014         fm10k_close_mbx_service(hw);
1015         fm10k_dev_stop(dev);
1016         fm10k_dev_queue_release(dev);
1017         fm10k_stop_hw(hw);
1018 }
1019
1020 static int
1021 fm10k_link_update(struct rte_eth_dev *dev,
1022         __rte_unused int wait_to_complete)
1023 {
1024         PMD_INIT_FUNC_TRACE();
1025
1026         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1027          * x8 PCIe interface. For now, we leave the speed undefined since there
1028          * is no 50Gbps Ethernet. */
1029         dev->data->dev_link.link_speed  = 0;
1030         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1031         dev->data->dev_link.link_status = 1;
1032
1033         return 0;
1034 }
1035
1036 static void
1037 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1038 {
1039         uint64_t ipackets, opackets, ibytes, obytes;
1040         struct fm10k_hw *hw =
1041                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1042         struct fm10k_hw_stats *hw_stats =
1043                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1044         int i;
1045
1046         PMD_INIT_FUNC_TRACE();
1047
1048         fm10k_update_hw_stats(hw, hw_stats);
1049
1050         ipackets = opackets = ibytes = obytes = 0;
1051         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1052                 (i < hw->mac.max_queues); ++i) {
1053                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1054                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1055                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1056                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1057                 ipackets += stats->q_ipackets[i];
1058                 opackets += stats->q_opackets[i];
1059                 ibytes   += stats->q_ibytes[i];
1060                 obytes   += stats->q_obytes[i];
1061         }
1062         stats->ipackets = ipackets;
1063         stats->opackets = opackets;
1064         stats->ibytes = ibytes;
1065         stats->obytes = obytes;
1066 }
1067
1068 static void
1069 fm10k_stats_reset(struct rte_eth_dev *dev)
1070 {
1071         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1072         struct fm10k_hw_stats *hw_stats =
1073                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         memset(hw_stats, 0, sizeof(*hw_stats));
1078         fm10k_rebind_hw_stats(hw, hw_stats);
1079 }
1080
1081 static void
1082 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1083         struct rte_eth_dev_info *dev_info)
1084 {
1085         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1086
1087         PMD_INIT_FUNC_TRACE();
1088
1089         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1090         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1091         dev_info->max_rx_queues      = hw->mac.max_queues;
1092         dev_info->max_tx_queues      = hw->mac.max_queues;
1093         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1094         dev_info->max_hash_mac_addrs = 0;
1095         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1096         dev_info->vmdq_pool_base     = 0;
1097         dev_info->vmdq_queue_base    = 0;
1098         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1099         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1100         dev_info->rx_offload_capa =
1101                 DEV_RX_OFFLOAD_VLAN_STRIP |
1102                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1103                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1104                 DEV_RX_OFFLOAD_TCP_CKSUM;
1105         dev_info->tx_offload_capa =
1106                 DEV_TX_OFFLOAD_VLAN_INSERT |
1107                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1108                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1109                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1110                 DEV_TX_OFFLOAD_TCP_TSO;
1111
1112         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1113         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1114
1115         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1116                 .rx_thresh = {
1117                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1118                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1119                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1120                 },
1121                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1122                 .rx_drop_en = 0,
1123         };
1124
1125         dev_info->default_txconf = (struct rte_eth_txconf) {
1126                 .tx_thresh = {
1127                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1128                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1129                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1130                 },
1131                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1132                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1133                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1134                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1135         };
1136
1137         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1138                 .nb_max = FM10K_MAX_RX_DESC,
1139                 .nb_min = FM10K_MIN_RX_DESC,
1140                 .nb_align = FM10K_MULT_RX_DESC,
1141         };
1142
1143         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1144                 .nb_max = FM10K_MAX_TX_DESC,
1145                 .nb_min = FM10K_MIN_TX_DESC,
1146                 .nb_align = FM10K_MULT_TX_DESC,
1147         };
1148 }
1149
1150 static int
1151 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1152 {
1153         s32 result;
1154         uint16_t mac_num = 0;
1155         uint32_t vid_idx, vid_bit, mac_index;
1156         struct fm10k_hw *hw;
1157         struct fm10k_macvlan_filter_info *macvlan;
1158         struct rte_eth_dev_data *data = dev->data;
1159
1160         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1161         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1162
1163         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1164                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1165                 return (-EINVAL);
1166         }
1167
1168         if (vlan_id > ETH_VLAN_ID_MAX) {
1169                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1170                 return (-EINVAL);
1171         }
1172
1173         vid_idx = FM10K_VFTA_IDX(vlan_id);
1174         vid_bit = FM10K_VFTA_BIT(vlan_id);
1175         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1176         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1177                 return 0;
1178         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1179         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1180                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1181                         "in the VLAN filter table");
1182                 return (-EINVAL);
1183         }
1184
1185         fm10k_mbx_lock(hw);
1186         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1187         fm10k_mbx_unlock(hw);
1188         if (result != FM10K_SUCCESS) {
1189                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1190                 return (-EIO);
1191         }
1192
1193         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1194                         (result == FM10K_SUCCESS); mac_index++) {
1195                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1196                         continue;
1197                 if (mac_num > macvlan->mac_num - 1) {
1198                         PMD_INIT_LOG(ERR, "MAC address number "
1199                                         "not match");
1200                         break;
1201                 }
1202                 fm10k_mbx_lock(hw);
1203                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1204                         data->mac_addrs[mac_index].addr_bytes,
1205                         vlan_id, on, 0);
1206                 fm10k_mbx_unlock(hw);
1207                 mac_num++;
1208         }
1209         if (result != FM10K_SUCCESS) {
1210                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1211                 return (-EIO);
1212         }
1213
1214         if (on) {
1215                 macvlan->vlan_num++;
1216                 macvlan->vfta[vid_idx] |= vid_bit;
1217         } else {
1218                 macvlan->vlan_num--;
1219                 macvlan->vfta[vid_idx] &= ~vid_bit;
1220         }
1221         return 0;
1222 }
1223
1224 static void
1225 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1226 {
1227         if (mask & ETH_VLAN_STRIP_MASK) {
1228                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1229                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1230                                         "always on in fm10k");
1231         }
1232
1233         if (mask & ETH_VLAN_EXTEND_MASK) {
1234                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1235                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1236                                         "supported in fm10k");
1237         }
1238
1239         if (mask & ETH_VLAN_FILTER_MASK) {
1240                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1241                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1242         }
1243 }
1244
1245 /* Add/Remove a MAC address, and update filters to main VSI */
1246 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1247                 const u8 *mac, bool add, uint32_t pool)
1248 {
1249         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1250         struct fm10k_macvlan_filter_info *macvlan;
1251         uint32_t i, j, k;
1252
1253         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1254
1255         if (pool != MAIN_VSI_POOL_NUMBER) {
1256                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1257                         "mac to pool %u", pool);
1258                 return;
1259         }
1260         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1261                 if (!macvlan->vfta[j])
1262                         continue;
1263                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1264                         if (!(macvlan->vfta[j] & (1 << k)))
1265                                 continue;
1266                         if (i + 1 > macvlan->vlan_num) {
1267                                 PMD_INIT_LOG(ERR, "vlan number not match");
1268                                 return;
1269                         }
1270                         fm10k_mbx_lock(hw);
1271                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1272                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1273                         fm10k_mbx_unlock(hw);
1274                         i++;
1275                 }
1276         }
1277 }
1278
1279 /* Add/Remove a MAC address, and update filters to VMDQ */
1280 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1281                 const u8 *mac, bool add, uint32_t pool)
1282 {
1283         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1284         struct fm10k_macvlan_filter_info *macvlan;
1285         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1286         uint32_t i;
1287
1288         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1289         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1290
1291         if (pool > macvlan->nb_queue_pools) {
1292                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1293                         " Max pool is %u",
1294                         pool, macvlan->nb_queue_pools);
1295                 return;
1296         }
1297         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1298                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1299                         continue;
1300                 fm10k_mbx_lock(hw);
1301                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1302                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1303                 fm10k_mbx_unlock(hw);
1304         }
1305 }
1306
1307 /* Add/Remove a MAC address, and update filters */
1308 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1309                 const u8 *mac, bool add, uint32_t pool)
1310 {
1311         struct fm10k_macvlan_filter_info *macvlan;
1312
1313         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1314
1315         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1316                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1317         else
1318                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1319
1320         if (add)
1321                 macvlan->mac_num++;
1322         else
1323                 macvlan->mac_num--;
1324 }
1325
1326 /* Add a MAC address, and update filters */
1327 static void
1328 fm10k_macaddr_add(struct rte_eth_dev *dev,
1329                 struct ether_addr *mac_addr,
1330                 uint32_t index,
1331                 uint32_t pool)
1332 {
1333         struct fm10k_macvlan_filter_info *macvlan;
1334
1335         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1336         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1337         macvlan->mac_vmdq_id[index] = pool;
1338 }
1339
1340 /* Remove a MAC address, and update filters */
1341 static void
1342 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1343 {
1344         struct rte_eth_dev_data *data = dev->data;
1345         struct fm10k_macvlan_filter_info *macvlan;
1346
1347         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1348         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1349                         FALSE, macvlan->mac_vmdq_id[index]);
1350         macvlan->mac_vmdq_id[index] = 0;
1351 }
1352
1353 static inline int
1354 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1355 {
1356         if ((request < min) || (request > max) || ((request % mult) != 0))
1357                 return -1;
1358         else
1359                 return 0;
1360 }
1361
1362 /*
1363  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1364  * the physical address is required. If the memzone is already created, then
1365  * this function returns a pointer to the existing memzone.
1366  */
1367 static inline const struct rte_memzone *
1368 allocate_hw_ring(const char *driver_name, const char *ring_name,
1369         uint8_t port_id, uint16_t queue_id, int socket_id,
1370         uint32_t size, uint32_t align)
1371 {
1372         char name[RTE_MEMZONE_NAMESIZE];
1373         const struct rte_memzone *mz;
1374
1375         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1376                  driver_name, ring_name, port_id, queue_id, socket_id);
1377
1378         /* return the memzone if it already exists */
1379         mz = rte_memzone_lookup(name);
1380         if (mz)
1381                 return mz;
1382
1383 #ifdef RTE_LIBRTE_XEN_DOM0
1384         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1385                                            RTE_PGSIZE_2M);
1386 #else
1387         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1388 #endif
1389 }
1390
1391 static inline int
1392 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1393 {
1394         if ((request < min) || (request > max) || ((div % request) != 0))
1395                 return -1;
1396         else
1397                 return 0;
1398 }
1399
1400 static inline int
1401 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1402 {
1403         uint16_t rx_free_thresh;
1404
1405         if (conf->rx_free_thresh == 0)
1406                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1407         else
1408                 rx_free_thresh = conf->rx_free_thresh;
1409
1410         /* make sure the requested threshold satisfies the constraints */
1411         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1412                         FM10K_RX_FREE_THRESH_MAX(q),
1413                         FM10K_RX_FREE_THRESH_DIV(q),
1414                         rx_free_thresh)) {
1415                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1416                         "less than or equal to %u, "
1417                         "greater than or equal to %u, "
1418                         "and a divisor of %u",
1419                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1420                         FM10K_RX_FREE_THRESH_MIN(q),
1421                         FM10K_RX_FREE_THRESH_DIV(q));
1422                 return (-EINVAL);
1423         }
1424
1425         q->alloc_thresh = rx_free_thresh;
1426         q->drop_en = conf->rx_drop_en;
1427         q->rx_deferred_start = conf->rx_deferred_start;
1428
1429         return 0;
1430 }
1431
1432 /*
1433  * Hardware requires specific alignment for Rx packet buffers. At
1434  * least one of the following two conditions must be satisfied.
1435  *  1. Address is 512B aligned
1436  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1437  *
1438  * As such, the driver may need to adjust the DMA address within the
1439  * buffer by up to 512B.
1440  *
1441  * return 1 if the element size is valid, otherwise return 0.
1442  */
1443 static int
1444 mempool_element_size_valid(struct rte_mempool *mp)
1445 {
1446         uint32_t min_size;
1447
1448         /* elt_size includes mbuf header and headroom */
1449         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1450                         RTE_PKTMBUF_HEADROOM;
1451
1452         /* account for up to 512B of alignment */
1453         min_size -= FM10K_RX_DATABUF_ALIGN;
1454
1455         /* sanity check for overflow */
1456         if (min_size > mp->elt_size)
1457                 return 0;
1458
1459         /* size is valid */
1460         return 1;
1461 }
1462
1463 static int
1464 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1465         uint16_t nb_desc, unsigned int socket_id,
1466         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1467 {
1468         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1469         struct fm10k_rx_queue *q;
1470         const struct rte_memzone *mz;
1471
1472         PMD_INIT_FUNC_TRACE();
1473
1474         /* make sure the mempool element size can account for alignment. */
1475         if (!mempool_element_size_valid(mp)) {
1476                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1477                 return (-EINVAL);
1478         }
1479
1480         /* make sure a valid number of descriptors have been requested */
1481         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1482                                 FM10K_MULT_RX_DESC, nb_desc)) {
1483                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1484                         "less than or equal to %"PRIu32", "
1485                         "greater than or equal to %u, "
1486                         "and a multiple of %u",
1487                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1488                         FM10K_MULT_RX_DESC);
1489                 return (-EINVAL);
1490         }
1491
1492         /*
1493          * if this queue existed already, free the associated memory. The
1494          * queue cannot be reused in case we need to allocate memory on
1495          * different socket than was previously used.
1496          */
1497         if (dev->data->rx_queues[queue_id] != NULL) {
1498                 rx_queue_free(dev->data->rx_queues[queue_id]);
1499                 dev->data->rx_queues[queue_id] = NULL;
1500         }
1501
1502         /* allocate memory for the queue structure */
1503         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1504                                 socket_id);
1505         if (q == NULL) {
1506                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1507                 return (-ENOMEM);
1508         }
1509
1510         /* setup queue */
1511         q->mp = mp;
1512         q->nb_desc = nb_desc;
1513         q->port_id = dev->data->port_id;
1514         q->queue_id = queue_id;
1515         q->tail_ptr = (volatile uint32_t *)
1516                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1517         if (handle_rxconf(q, conf))
1518                 return (-EINVAL);
1519
1520         /* allocate memory for the software ring */
1521         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1522                                         nb_desc * sizeof(struct rte_mbuf *),
1523                                         RTE_CACHE_LINE_SIZE, socket_id);
1524         if (q->sw_ring == NULL) {
1525                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1526                 rte_free(q);
1527                 return (-ENOMEM);
1528         }
1529
1530         /*
1531          * allocate memory for the hardware descriptor ring. A memzone large
1532          * enough to hold the maximum ring size is requested to allow for
1533          * resizing in later calls to the queue setup function.
1534          */
1535         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1536                                 dev->data->port_id, queue_id, socket_id,
1537                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1538         if (mz == NULL) {
1539                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1540                 rte_free(q->sw_ring);
1541                 rte_free(q);
1542                 return (-ENOMEM);
1543         }
1544         q->hw_ring = mz->addr;
1545 #ifdef RTE_LIBRTE_XEN_DOM0
1546         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1547 #else
1548         q->hw_ring_phys_addr = mz->phys_addr;
1549 #endif
1550
1551         dev->data->rx_queues[queue_id] = q;
1552         return 0;
1553 }
1554
1555 static void
1556 fm10k_rx_queue_release(void *queue)
1557 {
1558         PMD_INIT_FUNC_TRACE();
1559
1560         rx_queue_free(queue);
1561 }
1562
1563 static inline int
1564 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1565 {
1566         uint16_t tx_free_thresh;
1567         uint16_t tx_rs_thresh;
1568
1569         /* constraint MACROs require that tx_free_thresh is configured
1570          * before tx_rs_thresh */
1571         if (conf->tx_free_thresh == 0)
1572                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1573         else
1574                 tx_free_thresh = conf->tx_free_thresh;
1575
1576         /* make sure the requested threshold satisfies the constraints */
1577         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1578                         FM10K_TX_FREE_THRESH_MAX(q),
1579                         FM10K_TX_FREE_THRESH_DIV(q),
1580                         tx_free_thresh)) {
1581                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1582                         "less than or equal to %u, "
1583                         "greater than or equal to %u, "
1584                         "and a divisor of %u",
1585                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1586                         FM10K_TX_FREE_THRESH_MIN(q),
1587                         FM10K_TX_FREE_THRESH_DIV(q));
1588                 return (-EINVAL);
1589         }
1590
1591         q->free_thresh = tx_free_thresh;
1592
1593         if (conf->tx_rs_thresh == 0)
1594                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1595         else
1596                 tx_rs_thresh = conf->tx_rs_thresh;
1597
1598         q->tx_deferred_start = conf->tx_deferred_start;
1599
1600         /* make sure the requested threshold satisfies the constraints */
1601         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1602                         FM10K_TX_RS_THRESH_MAX(q),
1603                         FM10K_TX_RS_THRESH_DIV(q),
1604                         tx_rs_thresh)) {
1605                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1606                         "less than or equal to %u, "
1607                         "greater than or equal to %u, "
1608                         "and a divisor of %u",
1609                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1610                         FM10K_TX_RS_THRESH_MIN(q),
1611                         FM10K_TX_RS_THRESH_DIV(q));
1612                 return (-EINVAL);
1613         }
1614
1615         q->rs_thresh = tx_rs_thresh;
1616
1617         return 0;
1618 }
1619
1620 static int
1621 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1622         uint16_t nb_desc, unsigned int socket_id,
1623         const struct rte_eth_txconf *conf)
1624 {
1625         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1626         struct fm10k_tx_queue *q;
1627         const struct rte_memzone *mz;
1628
1629         PMD_INIT_FUNC_TRACE();
1630
1631         /* make sure a valid number of descriptors have been requested */
1632         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1633                                 FM10K_MULT_TX_DESC, nb_desc)) {
1634                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1635                         "less than or equal to %"PRIu32", "
1636                         "greater than or equal to %u, "
1637                         "and a multiple of %u",
1638                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1639                         FM10K_MULT_TX_DESC);
1640                 return (-EINVAL);
1641         }
1642
1643         /*
1644          * if this queue existed already, free the associated memory. The
1645          * queue cannot be reused in case we need to allocate memory on
1646          * different socket than was previously used.
1647          */
1648         if (dev->data->tx_queues[queue_id] != NULL) {
1649                 tx_queue_free(dev->data->tx_queues[queue_id]);
1650                 dev->data->tx_queues[queue_id] = NULL;
1651         }
1652
1653         /* allocate memory for the queue structure */
1654         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1655                                 socket_id);
1656         if (q == NULL) {
1657                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1658                 return (-ENOMEM);
1659         }
1660
1661         /* setup queue */
1662         q->nb_desc = nb_desc;
1663         q->port_id = dev->data->port_id;
1664         q->queue_id = queue_id;
1665         q->tail_ptr = (volatile uint32_t *)
1666                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1667         if (handle_txconf(q, conf))
1668                 return (-EINVAL);
1669
1670         /* allocate memory for the software ring */
1671         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1672                                         nb_desc * sizeof(struct rte_mbuf *),
1673                                         RTE_CACHE_LINE_SIZE, socket_id);
1674         if (q->sw_ring == NULL) {
1675                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1676                 rte_free(q);
1677                 return (-ENOMEM);
1678         }
1679
1680         /*
1681          * allocate memory for the hardware descriptor ring. A memzone large
1682          * enough to hold the maximum ring size is requested to allow for
1683          * resizing in later calls to the queue setup function.
1684          */
1685         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1686                                 dev->data->port_id, queue_id, socket_id,
1687                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1688         if (mz == NULL) {
1689                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1690                 rte_free(q->sw_ring);
1691                 rte_free(q);
1692                 return (-ENOMEM);
1693         }
1694         q->hw_ring = mz->addr;
1695 #ifdef RTE_LIBRTE_XEN_DOM0
1696         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1697 #else
1698         q->hw_ring_phys_addr = mz->phys_addr;
1699 #endif
1700
1701         /*
1702          * allocate memory for the RS bit tracker. Enough slots to hold the
1703          * descriptor index for each RS bit needing to be set are required.
1704          */
1705         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1706                                 ((nb_desc + 1) / q->rs_thresh) *
1707                                 sizeof(uint16_t),
1708                                 RTE_CACHE_LINE_SIZE, socket_id);
1709         if (q->rs_tracker.list == NULL) {
1710                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1711                 rte_free(q->sw_ring);
1712                 rte_free(q);
1713                 return (-ENOMEM);
1714         }
1715
1716         dev->data->tx_queues[queue_id] = q;
1717         return 0;
1718 }
1719
1720 static void
1721 fm10k_tx_queue_release(void *queue)
1722 {
1723         PMD_INIT_FUNC_TRACE();
1724
1725         tx_queue_free(queue);
1726 }
1727
1728 static int
1729 fm10k_reta_update(struct rte_eth_dev *dev,
1730                         struct rte_eth_rss_reta_entry64 *reta_conf,
1731                         uint16_t reta_size)
1732 {
1733         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1734         uint16_t i, j, idx, shift;
1735         uint8_t mask;
1736         uint32_t reta;
1737
1738         PMD_INIT_FUNC_TRACE();
1739
1740         if (reta_size > FM10K_MAX_RSS_INDICES) {
1741                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1742                         "(%d) doesn't match the number hardware can supported "
1743                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1744                 return -EINVAL;
1745         }
1746
1747         /*
1748          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1749          * 128-entries in 32 registers
1750          */
1751         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1752                 idx = i / RTE_RETA_GROUP_SIZE;
1753                 shift = i % RTE_RETA_GROUP_SIZE;
1754                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1755                                 BIT_MASK_PER_UINT32);
1756                 if (mask == 0)
1757                         continue;
1758
1759                 reta = 0;
1760                 if (mask != BIT_MASK_PER_UINT32)
1761                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1762
1763                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1764                         if (mask & (0x1 << j)) {
1765                                 if (mask != 0xF)
1766                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1767                                 reta |= reta_conf[idx].reta[shift + j] <<
1768                                                 (CHAR_BIT * j);
1769                         }
1770                 }
1771                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1772         }
1773
1774         return 0;
1775 }
1776
1777 static int
1778 fm10k_reta_query(struct rte_eth_dev *dev,
1779                         struct rte_eth_rss_reta_entry64 *reta_conf,
1780                         uint16_t reta_size)
1781 {
1782         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1783         uint16_t i, j, idx, shift;
1784         uint8_t mask;
1785         uint32_t reta;
1786
1787         PMD_INIT_FUNC_TRACE();
1788
1789         if (reta_size < FM10K_MAX_RSS_INDICES) {
1790                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1791                         "(%d) doesn't match the number hardware can supported "
1792                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1793                 return -EINVAL;
1794         }
1795
1796         /*
1797          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1798          * 128-entries in 32 registers
1799          */
1800         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1801                 idx = i / RTE_RETA_GROUP_SIZE;
1802                 shift = i % RTE_RETA_GROUP_SIZE;
1803                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1804                                 BIT_MASK_PER_UINT32);
1805                 if (mask == 0)
1806                         continue;
1807
1808                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1809                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1810                         if (mask & (0x1 << j))
1811                                 reta_conf[idx].reta[shift + j] = ((reta >>
1812                                         CHAR_BIT * j) & UINT8_MAX);
1813                 }
1814         }
1815
1816         return 0;
1817 }
1818
1819 static int
1820 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1821         struct rte_eth_rss_conf *rss_conf)
1822 {
1823         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1824         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1825         uint32_t mrqc;
1826         uint64_t hf = rss_conf->rss_hf;
1827         int i;
1828
1829         PMD_INIT_FUNC_TRACE();
1830
1831         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1832                 FM10K_RSSRK_ENTRIES_PER_REG)
1833                 return -EINVAL;
1834
1835         if (hf == 0)
1836                 return -EINVAL;
1837
1838         mrqc = 0;
1839         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1840         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1841         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1842         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1843         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1844         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1845         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1846         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1847         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1848
1849         /* If the mapping doesn't fit any supported, return */
1850         if (mrqc == 0)
1851                 return -EINVAL;
1852
1853         if (key != NULL)
1854                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1855                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1856
1857         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1858
1859         return 0;
1860 }
1861
1862 static int
1863 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1864         struct rte_eth_rss_conf *rss_conf)
1865 {
1866         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1867         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1868         uint32_t mrqc;
1869         uint64_t hf;
1870         int i;
1871
1872         PMD_INIT_FUNC_TRACE();
1873
1874         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1875                                 FM10K_RSSRK_ENTRIES_PER_REG)
1876                 return -EINVAL;
1877
1878         if (key != NULL)
1879                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1880                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1881
1882         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1883         hf = 0;
1884         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1885         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1886         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1887         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1888         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1889         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1890         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1891         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1892         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1893
1894         rss_conf->rss_hf = hf;
1895
1896         return 0;
1897 }
1898
1899 static void
1900 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1901 {
1902         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1903         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1904
1905         /* Bind all local non-queue interrupt to vector 0 */
1906         int_map |= 0;
1907
1908         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1909         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1910         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1911         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1912         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1913         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1914
1915         /* Enable misc causes */
1916         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1917                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1918                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1919                                 FM10K_EIMR_ENABLE(MAILBOX) |
1920                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1921                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1922                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1923                                 FM10K_EIMR_ENABLE(VFLR));
1924
1925         /* Enable ITR 0 */
1926         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1927                                         FM10K_ITR_MASK_CLEAR);
1928         FM10K_WRITE_FLUSH(hw);
1929 }
1930
1931 static void
1932 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1933 {
1934         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1935         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1936
1937         int_map |= 0;
1938
1939         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1940         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1941         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1942         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1943         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1944         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1945
1946         /* Disable misc causes */
1947         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1948                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1949                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1950                                 FM10K_EIMR_DISABLE(MAILBOX) |
1951                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1952                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1953                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1954                                 FM10K_EIMR_DISABLE(VFLR));
1955
1956         /* Disable ITR 0 */
1957         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1958         FM10K_WRITE_FLUSH(hw);
1959 }
1960
1961 static void
1962 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1963 {
1964         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1965         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1966
1967         /* Bind all local non-queue interrupt to vector 0 */
1968         int_map |= 0;
1969
1970         /* Only INT 0 available, other 15 are reserved. */
1971         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1972
1973         /* Enable ITR 0 */
1974         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1975                                         FM10K_ITR_MASK_CLEAR);
1976         FM10K_WRITE_FLUSH(hw);
1977 }
1978
1979 static void
1980 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1981 {
1982         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1983         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1984
1985         int_map |= 0;
1986
1987         /* Only INT 0 available, other 15 are reserved. */
1988         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1989
1990         /* Disable ITR 0 */
1991         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1992         FM10K_WRITE_FLUSH(hw);
1993 }
1994
1995 static int
1996 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1997 {
1998         struct fm10k_fault fault;
1999         int err;
2000         const char *estr = "Unknown error";
2001
2002         /* Process PCA fault */
2003         if (eicr & FM10K_EICR_PCA_FAULT) {
2004                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2005                 if (err)
2006                         goto error;
2007                 switch (fault.type) {
2008                 case PCA_NO_FAULT:
2009                         estr = "PCA_NO_FAULT"; break;
2010                 case PCA_UNMAPPED_ADDR:
2011                         estr = "PCA_UNMAPPED_ADDR"; break;
2012                 case PCA_BAD_QACCESS_PF:
2013                         estr = "PCA_BAD_QACCESS_PF"; break;
2014                 case PCA_BAD_QACCESS_VF:
2015                         estr = "PCA_BAD_QACCESS_VF"; break;
2016                 case PCA_MALICIOUS_REQ:
2017                         estr = "PCA_MALICIOUS_REQ"; break;
2018                 case PCA_POISONED_TLP:
2019                         estr = "PCA_POISONED_TLP"; break;
2020                 case PCA_TLP_ABORT:
2021                         estr = "PCA_TLP_ABORT"; break;
2022                 default:
2023                         goto error;
2024                 }
2025                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2026                         estr, fault.func ? "VF" : "PF", fault.func,
2027                         fault.address, fault.specinfo);
2028         }
2029
2030         /* Process THI fault */
2031         if (eicr & FM10K_EICR_THI_FAULT) {
2032                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2033                 if (err)
2034                         goto error;
2035                 switch (fault.type) {
2036                 case THI_NO_FAULT:
2037                         estr = "THI_NO_FAULT"; break;
2038                 case THI_MAL_DIS_Q_FAULT:
2039                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2040                 default:
2041                         goto error;
2042                 }
2043                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2044                         estr, fault.func ? "VF" : "PF", fault.func,
2045                         fault.address, fault.specinfo);
2046         }
2047
2048         /* Process FUM fault */
2049         if (eicr & FM10K_EICR_FUM_FAULT) {
2050                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2051                 if (err)
2052                         goto error;
2053                 switch (fault.type) {
2054                 case FUM_NO_FAULT:
2055                         estr = "FUM_NO_FAULT"; break;
2056                 case FUM_UNMAPPED_ADDR:
2057                         estr = "FUM_UNMAPPED_ADDR"; break;
2058                 case FUM_POISONED_TLP:
2059                         estr = "FUM_POISONED_TLP"; break;
2060                 case FUM_BAD_VF_QACCESS:
2061                         estr = "FUM_BAD_VF_QACCESS"; break;
2062                 case FUM_ADD_DECODE_ERR:
2063                         estr = "FUM_ADD_DECODE_ERR"; break;
2064                 case FUM_RO_ERROR:
2065                         estr = "FUM_RO_ERROR"; break;
2066                 case FUM_QPRC_CRC_ERROR:
2067                         estr = "FUM_QPRC_CRC_ERROR"; break;
2068                 case FUM_CSR_TIMEOUT:
2069                         estr = "FUM_CSR_TIMEOUT"; break;
2070                 case FUM_INVALID_TYPE:
2071                         estr = "FUM_INVALID_TYPE"; break;
2072                 case FUM_INVALID_LENGTH:
2073                         estr = "FUM_INVALID_LENGTH"; break;
2074                 case FUM_INVALID_BE:
2075                         estr = "FUM_INVALID_BE"; break;
2076                 case FUM_INVALID_ALIGN:
2077                         estr = "FUM_INVALID_ALIGN"; break;
2078                 default:
2079                         goto error;
2080                 }
2081                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2082                         estr, fault.func ? "VF" : "PF", fault.func,
2083                         fault.address, fault.specinfo);
2084         }
2085
2086         return 0;
2087 error:
2088         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2089         return err;
2090 }
2091
2092 /**
2093  * PF interrupt handler triggered by NIC for handling specific interrupt.
2094  *
2095  * @param handle
2096  *  Pointer to interrupt handle.
2097  * @param param
2098  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2099  *
2100  * @return
2101  *  void
2102  */
2103 static void
2104 fm10k_dev_interrupt_handler_pf(
2105                         __rte_unused struct rte_intr_handle *handle,
2106                         void *param)
2107 {
2108         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2109         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2110         uint32_t cause, status;
2111
2112         if (hw->mac.type != fm10k_mac_pf)
2113                 return;
2114
2115         cause = FM10K_READ_REG(hw, FM10K_EICR);
2116
2117         /* Handle PCI fault cases */
2118         if (cause & FM10K_EICR_FAULT_MASK) {
2119                 PMD_INIT_LOG(ERR, "INT: find fault!");
2120                 fm10k_dev_handle_fault(hw, cause);
2121         }
2122
2123         /* Handle switch up/down */
2124         if (cause & FM10K_EICR_SWITCHNOTREADY)
2125                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2126
2127         if (cause & FM10K_EICR_SWITCHREADY)
2128                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2129
2130         /* Handle mailbox message */
2131         fm10k_mbx_lock(hw);
2132         hw->mbx.ops.process(hw, &hw->mbx);
2133         fm10k_mbx_unlock(hw);
2134
2135         /* Handle SRAM error */
2136         if (cause & FM10K_EICR_SRAMERROR) {
2137                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2138
2139                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2140                 /* Write to clear pending bits */
2141                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2142
2143                 /* Todo: print out error message after shared code  updates */
2144         }
2145
2146         /* Clear these 3 events if having any */
2147         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2148                  FM10K_EICR_SWITCHREADY;
2149         if (cause)
2150                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2151
2152         /* Re-enable interrupt from device side */
2153         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2154                                         FM10K_ITR_MASK_CLEAR);
2155         /* Re-enable interrupt from host side */
2156         rte_intr_enable(&(dev->pci_dev->intr_handle));
2157 }
2158
2159 /**
2160  * VF interrupt handler triggered by NIC for handling specific interrupt.
2161  *
2162  * @param handle
2163  *  Pointer to interrupt handle.
2164  * @param param
2165  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2166  *
2167  * @return
2168  *  void
2169  */
2170 static void
2171 fm10k_dev_interrupt_handler_vf(
2172                         __rte_unused struct rte_intr_handle *handle,
2173                         void *param)
2174 {
2175         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2176         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2177
2178         if (hw->mac.type != fm10k_mac_vf)
2179                 return;
2180
2181         /* Handle mailbox message if lock is acquired */
2182         fm10k_mbx_lock(hw);
2183         hw->mbx.ops.process(hw, &hw->mbx);
2184         fm10k_mbx_unlock(hw);
2185
2186         /* Re-enable interrupt from device side */
2187         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2188                                         FM10K_ITR_MASK_CLEAR);
2189         /* Re-enable interrupt from host side */
2190         rte_intr_enable(&(dev->pci_dev->intr_handle));
2191 }
2192
2193 /* Mailbox message handler in VF */
2194 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2195         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2196         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2197         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2198         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2199 };
2200
2201 /* Mailbox message handler in PF */
2202 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2203         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2204         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2205         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2206         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2207         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2208         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2209         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2210 };
2211
2212 static int
2213 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2214 {
2215         int err;
2216
2217         /* Initialize mailbox lock */
2218         fm10k_mbx_initlock(hw);
2219
2220         /* Replace default message handler with new ones */
2221         if (hw->mac.type == fm10k_mac_pf)
2222                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2223         else
2224                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2225
2226         if (err) {
2227                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2228                                 err);
2229                 return err;
2230         }
2231         /* Connect to SM for PF device or PF for VF device */
2232         return hw->mbx.ops.connect(hw, &hw->mbx);
2233 }
2234
2235 static void
2236 fm10k_close_mbx_service(struct fm10k_hw *hw)
2237 {
2238         /* Disconnect from SM for PF device or PF for VF device */
2239         hw->mbx.ops.disconnect(hw, &hw->mbx);
2240 }
2241
2242 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2243         .dev_configure          = fm10k_dev_configure,
2244         .dev_start              = fm10k_dev_start,
2245         .dev_stop               = fm10k_dev_stop,
2246         .dev_close              = fm10k_dev_close,
2247         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2248         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2249         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2250         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2251         .stats_get              = fm10k_stats_get,
2252         .stats_reset            = fm10k_stats_reset,
2253         .link_update            = fm10k_link_update,
2254         .dev_infos_get          = fm10k_dev_infos_get,
2255         .vlan_filter_set        = fm10k_vlan_filter_set,
2256         .vlan_offload_set       = fm10k_vlan_offload_set,
2257         .mac_addr_add           = fm10k_macaddr_add,
2258         .mac_addr_remove        = fm10k_macaddr_remove,
2259         .rx_queue_start         = fm10k_dev_rx_queue_start,
2260         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2261         .tx_queue_start         = fm10k_dev_tx_queue_start,
2262         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2263         .rx_queue_setup         = fm10k_rx_queue_setup,
2264         .rx_queue_release       = fm10k_rx_queue_release,
2265         .tx_queue_setup         = fm10k_tx_queue_setup,
2266         .tx_queue_release       = fm10k_tx_queue_release,
2267         .reta_update            = fm10k_reta_update,
2268         .reta_query             = fm10k_reta_query,
2269         .rss_hash_update        = fm10k_rss_hash_update,
2270         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2271 };
2272
2273 static int
2274 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2275 {
2276         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2277         int diag;
2278         struct fm10k_macvlan_filter_info *macvlan;
2279
2280         PMD_INIT_FUNC_TRACE();
2281
2282         dev->dev_ops = &fm10k_eth_dev_ops;
2283         dev->rx_pkt_burst = &fm10k_recv_pkts;
2284         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2285
2286         if (dev->data->scattered_rx)
2287                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2288
2289         /* only initialize in the primary process */
2290         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2291                 return 0;
2292
2293         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2294         memset(macvlan, 0, sizeof(*macvlan));
2295         /* Vendor and Device ID need to be set before init of shared code */
2296         memset(hw, 0, sizeof(*hw));
2297         hw->device_id = dev->pci_dev->id.device_id;
2298         hw->vendor_id = dev->pci_dev->id.vendor_id;
2299         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2300         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2301         hw->revision_id = 0;
2302         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2303         if (hw->hw_addr == NULL) {
2304                 PMD_INIT_LOG(ERR, "Bad mem resource."
2305                         " Try to blacklist unused devices.");
2306                 return -EIO;
2307         }
2308
2309         /* Store fm10k_adapter pointer */
2310         hw->back = dev->data->dev_private;
2311
2312         /* Initialize the shared code */
2313         diag = fm10k_init_shared_code(hw);
2314         if (diag != FM10K_SUCCESS) {
2315                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2316                 return -EIO;
2317         }
2318
2319         /*
2320          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2321          * there is no way to get link status without reading BAR4.  Until this
2322          * works, assume we have maximum bandwidth.
2323          * @todo - fix bus info
2324          */
2325         hw->bus_caps.speed = fm10k_bus_speed_8000;
2326         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2327         hw->bus_caps.payload = fm10k_bus_payload_512;
2328         hw->bus.speed = fm10k_bus_speed_8000;
2329         hw->bus.width = fm10k_bus_width_pcie_x8;
2330         hw->bus.payload = fm10k_bus_payload_256;
2331
2332         /* Initialize the hw */
2333         diag = fm10k_init_hw(hw);
2334         if (diag != FM10K_SUCCESS) {
2335                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2336                 return -EIO;
2337         }
2338
2339         /* Initialize MAC address(es) */
2340         dev->data->mac_addrs = rte_zmalloc("fm10k",
2341                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2342         if (dev->data->mac_addrs == NULL) {
2343                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2344                 return -ENOMEM;
2345         }
2346
2347         diag = fm10k_read_mac_addr(hw);
2348
2349         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2350                         &dev->data->mac_addrs[0]);
2351
2352         if (diag != FM10K_SUCCESS ||
2353                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2354
2355                 /* Generate a random addr */
2356                 eth_random_addr(hw->mac.addr);
2357                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2358                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2359                 &dev->data->mac_addrs[0]);
2360         }
2361
2362         /* Reset the hw statistics */
2363         fm10k_stats_reset(dev);
2364
2365         /* Reset the hw */
2366         diag = fm10k_reset_hw(hw);
2367         if (diag != FM10K_SUCCESS) {
2368                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2369                 return -EIO;
2370         }
2371
2372         /* Setup mailbox service */
2373         diag = fm10k_setup_mbx_service(hw);
2374         if (diag != FM10K_SUCCESS) {
2375                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2376                 return -EIO;
2377         }
2378
2379         /*PF/VF has different interrupt handling mechanism */
2380         if (hw->mac.type == fm10k_mac_pf) {
2381                 /* register callback func to eal lib */
2382                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2383                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2384
2385                 /* enable MISC interrupt */
2386                 fm10k_dev_enable_intr_pf(dev);
2387         } else { /* VF */
2388                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2389                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2390
2391                 fm10k_dev_enable_intr_vf(dev);
2392         }
2393
2394         /* Enable uio intr after callback registered */
2395         rte_intr_enable(&(dev->pci_dev->intr_handle));
2396
2397         hw->mac.ops.update_int_moderator(hw);
2398
2399         /* Make sure Switch Manager is ready before going forward. */
2400         if (hw->mac.type == fm10k_mac_pf) {
2401                 int switch_ready = 0;
2402                 int i;
2403
2404                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2405                         fm10k_mbx_lock(hw);
2406                         hw->mac.ops.get_host_state(hw, &switch_ready);
2407                         fm10k_mbx_unlock(hw);
2408                         if (switch_ready)
2409                                 break;
2410                         /* Delay some time to acquire async LPORT_MAP info. */
2411                         rte_delay_us(WAIT_SWITCH_MSG_US);
2412                 }
2413
2414                 if (switch_ready == 0) {
2415                         PMD_INIT_LOG(ERR, "switch is not ready");
2416                         return -1;
2417                 }
2418         }
2419
2420         /*
2421          * Below function will trigger operations on mailbox, acquire lock to
2422          * avoid race condition from interrupt handler. Operations on mailbox
2423          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2424          * will handle and generate an interrupt to our side. Then,  FIFO in
2425          * mailbox will be touched.
2426          */
2427         fm10k_mbx_lock(hw);
2428         /* Enable port first */
2429         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2430
2431         /* Set unicast mode by default. App can change to other mode in other
2432          * API func.
2433          */
2434         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2435                                         FM10K_XCAST_MODE_NONE);
2436
2437         fm10k_mbx_unlock(hw);
2438
2439         /* Add default mac address */
2440         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2441                 MAIN_VSI_POOL_NUMBER);
2442
2443         return 0;
2444 }
2445
2446 static int
2447 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2448 {
2449         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2450
2451         PMD_INIT_FUNC_TRACE();
2452
2453         /* only uninitialize in the primary process */
2454         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2455                 return 0;
2456
2457         /* safe to close dev here */
2458         fm10k_dev_close(dev);
2459
2460         dev->dev_ops = NULL;
2461         dev->rx_pkt_burst = NULL;
2462         dev->tx_pkt_burst = NULL;
2463
2464         /* disable uio/vfio intr */
2465         rte_intr_disable(&(dev->pci_dev->intr_handle));
2466
2467         /*PF/VF has different interrupt handling mechanism */
2468         if (hw->mac.type == fm10k_mac_pf) {
2469                 /* disable interrupt */
2470                 fm10k_dev_disable_intr_pf(dev);
2471
2472                 /* unregister callback func to eal lib */
2473                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2474                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2475         } else {
2476                 /* disable interrupt */
2477                 fm10k_dev_disable_intr_vf(dev);
2478
2479                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2480                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2481         }
2482
2483         /* free mac memory */
2484         if (dev->data->mac_addrs) {
2485                 rte_free(dev->data->mac_addrs);
2486                 dev->data->mac_addrs = NULL;
2487         }
2488
2489         memset(hw, 0, sizeof(*hw));
2490
2491         return 0;
2492 }
2493
2494 /*
2495  * The set of PCI devices this driver supports. This driver will enable both PF
2496  * and SRIOV-VF devices.
2497  */
2498 static const struct rte_pci_id pci_id_fm10k_map[] = {
2499 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2500 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2501 #include "rte_pci_dev_ids.h"
2502         { .vendor_id = 0, /* sentinel */ },
2503 };
2504
2505 static struct eth_driver rte_pmd_fm10k = {
2506         .pci_drv = {
2507                 .name = "rte_pmd_fm10k",
2508                 .id_table = pci_id_fm10k_map,
2509                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2510         },
2511         .eth_dev_init = eth_fm10k_dev_init,
2512         .eth_dev_uninit = eth_fm10k_dev_uninit,
2513         .dev_private_size = sizeof(struct fm10k_adapter),
2514 };
2515
2516 /*
2517  * Driver initialization routine.
2518  * Invoked once at EAL init time.
2519  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2520  */
2521 static int
2522 rte_pmd_fm10k_init(__rte_unused const char *name,
2523         __rte_unused const char *params)
2524 {
2525         PMD_INIT_FUNC_TRACE();
2526         rte_eth_driver_register(&rte_pmd_fm10k);
2527         return 0;
2528 }
2529
2530 static struct rte_driver rte_fm10k_driver = {
2531         .type = PMD_PDEV,
2532         .init = rte_pmd_fm10k_init,
2533 };
2534
2535 PMD_REGISTER_DRIVER(rte_fm10k_driver);