drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  64 static int
  65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  67         const u8 *mac, bool add, uint32_t pool);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         return 0;
 125 }
 126
 127 /*
 128  * clean queue, descriptor rings, free software buffers used when stopping
 129  * device.
 130  */
 131 static inline void
 132 rx_queue_clean(struct fm10k_rx_queue *q)
 133 {
 134         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 135         uint32_t i;
 136         PMD_INIT_FUNC_TRACE();
 137
 138         /* zero descriptor rings */
 139         for (i = 0; i < q->nb_desc; ++i)
 140                 q->hw_ring[i] = zero;
 141
 142         /* free software buffers */
 143         for (i = 0; i < q->nb_desc; ++i) {
 144                 if (q->sw_ring[i]) {
 145                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 146                         q->sw_ring[i] = NULL;
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * free all queue memory used when releasing the queue (i.e. configure)
 153  */
 154 static inline void
 155 rx_queue_free(struct fm10k_rx_queue *q)
 156 {
 157         PMD_INIT_FUNC_TRACE();
 158         if (q) {
 159                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 160                 rx_queue_clean(q);
 161                 if (q->sw_ring) {
 162                         rte_free(q->sw_ring);
 163                         q->sw_ring = NULL;
 164                 }
 165                 rte_free(q);
 166                 q = NULL;
 167         }
 168 }
 169
 170 /*
 171  * disable RX queue, wait unitl HW finished necessary flush operation
 172  */
 173 static inline int
 174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 175 {
 176         uint32_t reg, i;
 177
 178         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 179         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 180                         reg & ~FM10K_RXQCTL_ENABLE);
 181
 182         /* Wait 100us at most */
 183         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 184                 rte_delay_us(1);
 185                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 186                 if (!(reg & FM10K_RXQCTL_ENABLE))
 187                         break;
 188         }
 189
 190         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 191                 return -1;
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * reset queue to initial state, allocate software buffers used when starting
 198  * device
 199  */
 200 static inline void
 201 tx_queue_reset(struct fm10k_tx_queue *q)
 202 {
 203         PMD_INIT_FUNC_TRACE();
 204         q->last_free = 0;
 205         q->next_free = 0;
 206         q->nb_used = 0;
 207         q->nb_free = q->nb_desc - 1;
 208         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 209         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 210 }
 211
 212 /*
 213  * clean queue, descriptor rings, free software buffers used when stopping
 214  * device
 215  */
 216 static inline void
 217 tx_queue_clean(struct fm10k_tx_queue *q)
 218 {
 219         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 220         uint32_t i;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         /* zero descriptor rings */
 224         for (i = 0; i < q->nb_desc; ++i)
 225                 q->hw_ring[i] = zero;
 226
 227         /* free software buffers */
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 if (q->sw_ring[i]) {
 230                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 231                         q->sw_ring[i] = NULL;
 232                 }
 233         }
 234 }
 235
 236 /*
 237  * free all queue memory used when releasing the queue (i.e. configure)
 238  */
 239 static inline void
 240 tx_queue_free(struct fm10k_tx_queue *q)
 241 {
 242         PMD_INIT_FUNC_TRACE();
 243         if (q) {
 244                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 245                 tx_queue_clean(q);
 246                 if (q->rs_tracker.list) {
 247                         rte_free(q->rs_tracker.list);
 248                         q->rs_tracker.list = NULL;
 249                 }
 250                 if (q->sw_ring) {
 251                         rte_free(q->sw_ring);
 252                         q->sw_ring = NULL;
 253                 }
 254                 rte_free(q);
 255                 q = NULL;
 256         }
 257 }
 258
 259 /*
 260  * disable TX queue, wait unitl HW finished necessary flush operation
 261  */
 262 static inline int
 263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 264 {
 265         uint32_t reg, i;
 266
 267         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 268         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 269                         reg & ~FM10K_TXDCTL_ENABLE);
 270
 271         /* Wait 100us at most */
 272         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 273                 rte_delay_us(1);
 274                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 275                 if (!(reg & FM10K_TXDCTL_ENABLE))
 276                         break;
 277         }
 278
 279         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 280                 return -1;
 281
 282         return 0;
 283 }
 284
 285 static int
 286 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 287 {
 288         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 291         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 292
 293         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 294
 295         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 296                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 297                 return -EINVAL;
 298         }
 299
 300         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 301                 return 0;
 302
 303         if (hw->mac.type == fm10k_mac_vf) {
 304                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 305                 return -EINVAL;
 306         }
 307
 308         /* Check VMDQ queue pool number */
 309         if (vmdq_conf->nb_queue_pools >
 310                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 311                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 312                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 313                         vmdq_conf->nb_queue_pools);
 314                 return -EINVAL;
 315         }
 316
 317         return 0;
 318 }
 319
 320 static int
 321 fm10k_dev_configure(struct rte_eth_dev *dev)
 322 {
 323         int ret;
 324
 325         PMD_INIT_FUNC_TRACE();
 326
 327         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 328                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 329         /* multipe queue mode checking */
 330         ret  = fm10k_check_mq_mode(dev);
 331         if (ret != 0) {
 332                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 333                             ret);
 334                 return ret;
 335         }
 336
 337         return 0;
 338 }
 339
 340 /* fls = find last set bit = 32 minus the number of leading zeros */
 341 #ifndef fls
 342 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 343 #endif
 344
 345 static void
 346 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 347 {
 348         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 349         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 350         uint32_t i;
 351
 352         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 353
 354         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 355                 if (!vmdq_conf->pool_map[i].pools)
 356                         continue;
 357                 fm10k_mbx_lock(hw);
 358                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 359                 fm10k_mbx_unlock(hw);
 360         }
 361 }
 362
 363 static void
 364 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 365 {
 366         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 367
 368         /* Add default mac address */
 369         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 370                 MAIN_VSI_POOL_NUMBER);
 371 }
 372
 373 static void
 374 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 375 {
 376         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 377         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 378         uint32_t mrqc, *key, i, reta, j;
 379         uint64_t hf;
 380
 381 #define RSS_KEY_SIZE 40
 382         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 383                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 384                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 385                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 386                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 387                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 388         };
 389
 390         if (dev->data->nb_rx_queues == 1 ||
 391             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 392             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 393                 return;
 394
 395         /* random key is rss_intel_key (default) or user provided (rss_key) */
 396         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 397                 key = (uint32_t *)rss_intel_key;
 398         else
 399                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 400
 401         /* Now fill our hash function seeds, 4 bytes at a time */
 402         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 403                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 404
 405         /*
 406          * Fill in redirection table
 407          * The byte-swap is needed because NIC registers are in
 408          * little-endian order.
 409          */
 410         reta = 0;
 411         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 412                 if (j == dev->data->nb_rx_queues)
 413                         j = 0;
 414                 reta = (reta << CHAR_BIT) | j;
 415                 if ((i & 3) == 3)
 416                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 417                                         rte_bswap32(reta));
 418         }
 419
 420         /*
 421          * Generate RSS hash based on packet types, TCP/UDP
 422          * port numbers and/or IPv4/v6 src and dst addresses
 423          */
 424         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 425         mrqc = 0;
 426         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 427         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 428         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 429         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 430         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 431         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 432         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 433         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 434         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 435
 436         if (mrqc == 0) {
 437                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 438                         "supported", hf);
 439                 return;
 440         }
 441
 442         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 443 }
 444
 445 static void
 446 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 447         uint16_t nb_lport_old, uint16_t nb_lport_new)
 448 {
 449         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 450         uint32_t i;
 451
 452         fm10k_mbx_lock(hw);
 453         /* Disable previous logic ports */
 454         if (nb_lport_old)
 455                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 456                         nb_lport_old, false);
 457         /* Enable new logic ports */
 458         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 459                 nb_lport_new, true);
 460         fm10k_mbx_unlock(hw);
 461
 462         for (i = 0; i < nb_lport_new; i++) {
 463                 /* Set unicast mode by default. App can change
 464                  * to other mode in other API func.
 465                  */
 466                 fm10k_mbx_lock(hw);
 467                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 468                         FM10K_XCAST_MODE_NONE);
 469                 fm10k_mbx_unlock(hw);
 470         }
 471 }
 472
 473 static void
 474 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 475 {
 476         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 477         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 478         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 479         struct fm10k_macvlan_filter_info *macvlan;
 480         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 481         uint16_t nb_lport_new, nb_lport_old;
 482
 483         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 484         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 485
 486         fm10k_dev_rss_configure(dev);
 487
 488         /* only PF supports VMDQ */
 489         if (hw->mac.type != fm10k_mac_pf)
 490                 return;
 491
 492         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 493                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 494
 495         /* no pool number change, no need to update logic port and VLAN/MAC */
 496         if (macvlan->nb_queue_pools == nb_queue_pools)
 497                 return;
 498
 499         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 500         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 501         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 502
 503         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 504         memset(dev->data->mac_addrs, 0,
 505                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 506         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 507                 &dev->data->mac_addrs[0]);
 508         memset(macvlan, 0, sizeof(*macvlan));
 509         macvlan->nb_queue_pools = nb_queue_pools;
 510
 511         if (nb_queue_pools)
 512                 fm10k_dev_vmdq_rx_configure(dev);
 513         else
 514                 fm10k_dev_pf_main_vsi_reset(dev);
 515 }
 516
 517 static int
 518 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 519 {
 520         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 521         int i, ret;
 522         struct fm10k_tx_queue *txq;
 523         uint64_t base_addr;
 524         uint32_t size;
 525
 526         /* Disable TXINT to avoid possible interrupt */
 527         for (i = 0; i < hw->mac.max_queues; i++)
 528                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 529                                 3 << FM10K_TXINT_TIMER_SHIFT);
 530
 531         /* Setup TX queue */
 532         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 533                 txq = dev->data->tx_queues[i];
 534                 base_addr = txq->hw_ring_phys_addr;
 535                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 536
 537                 /* disable queue to avoid issues while updating state */
 538                 ret = tx_queue_disable(hw, i);
 539                 if (ret) {
 540                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 541                         return -1;
 542                 }
 543
 544                 /* set location and size for descriptor ring */
 545                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 546                                 base_addr & UINT64_LOWER_32BITS_MASK);
 547                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 548                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 549                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 550         }
 551         return 0;
 552 }
 553
 554 static int
 555 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 556 {
 557         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 558         int i, ret;
 559         struct fm10k_rx_queue *rxq;
 560         uint64_t base_addr;
 561         uint32_t size;
 562         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 563         uint16_t buf_size;
 564
 565         /* Disable RXINT to avoid possible interrupt */
 566         for (i = 0; i < hw->mac.max_queues; i++)
 567                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 568                                 3 << FM10K_RXINT_TIMER_SHIFT);
 569
 570         /* Setup RX queues */
 571         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 572                 rxq = dev->data->rx_queues[i];
 573                 base_addr = rxq->hw_ring_phys_addr;
 574                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 575
 576                 /* disable queue to avoid issues while updating state */
 577                 ret = rx_queue_disable(hw, i);
 578                 if (ret) {
 579                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 580                         return -1;
 581                 }
 582
 583                 /* Setup the Base and Length of the Rx Descriptor Ring */
 584                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 585                                 base_addr & UINT64_LOWER_32BITS_MASK);
 586                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 587                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 588                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 589
 590                 /* Configure the Rx buffer size for one buff without split */
 591                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 592                         RTE_PKTMBUF_HEADROOM);
 593                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 594                  * reserved for this purpose, and the worst case could be 511B.
 595                  * But SRR reg assumes all buffers have the same size. In order
 596                  * to fill the gap, we'll have to consider the worst case and
 597                  * assume 512B is reserved. If we don't do so, it's possible
 598                  * for HW to overwrite data to next mbuf.
 599                  */
 600                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 601
 602                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 603                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 604
 605                 /* It adds dual VLAN length for supporting dual VLAN */
 606                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 607                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 608                         dev->data->dev_conf.rxmode.enable_scatter) {
 609                         uint32_t reg;
 610                         dev->data->scattered_rx = 1;
 611                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 612                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 613                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 614                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 615                 }
 616
 617                 /* Enable drop on empty, it's RO for VF */
 618                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 619                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 620
 621                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 622                 FM10K_WRITE_FLUSH(hw);
 623         }
 624
 625         /* Configure VMDQ/RSS if applicable */
 626         fm10k_dev_mq_rx_configure(dev);
 627         return 0;
 628 }
 629
 630 static int
 631 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 632 {
 633         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 634         int err = -1;
 635         uint32_t reg;
 636         struct fm10k_rx_queue *rxq;
 637
 638         PMD_INIT_FUNC_TRACE();
 639
 640         if (rx_queue_id < dev->data->nb_rx_queues) {
 641                 rxq = dev->data->rx_queues[rx_queue_id];
 642                 err = rx_queue_reset(rxq);
 643                 if (err == -ENOMEM) {
 644                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 645                         return err;
 646                 } else if (err == -EINVAL) {
 647                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 648                                 " %d", err);
 649                         return err;
 650                 }
 651
 652                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 653                  * Note: this must be done AFTER the queue is enabled on real
 654                  * hardware, but BEFORE the queue is enabled when using the
 655                  * emulation platform. Do it in both places for now and remove
 656                  * this comment and the following two register writes when the
 657                  * emulation platform is no longer being used.
 658                  */
 659                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 660                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 661
 662                 /* Set PF ownership flag for PF devices */
 663                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 664                 if (hw->mac.type == fm10k_mac_pf)
 665                         reg |= FM10K_RXQCTL_PF;
 666                 reg |= FM10K_RXQCTL_ENABLE;
 667                 /* enable RX queue */
 668                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 669                 FM10K_WRITE_FLUSH(hw);
 670
 671                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 672                  * Note: this must be done AFTER the queue is enabled
 673                  */
 674                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 675                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 676         }
 677
 678         return err;
 679 }
 680
 681 static int
 682 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 683 {
 684         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 685
 686         PMD_INIT_FUNC_TRACE();
 687
 688         if (rx_queue_id < dev->data->nb_rx_queues) {
 689                 /* Disable RX queue */
 690                 rx_queue_disable(hw, rx_queue_id);
 691
 692                 /* Free mbuf and clean HW ring */
 693                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 694         }
 695
 696         return 0;
 697 }
 698
 699 static int
 700 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 701 {
 702         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 703         /** @todo - this should be defined in the shared code */
 704 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 705         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 706         int err = 0;
 707
 708         PMD_INIT_FUNC_TRACE();
 709
 710         if (tx_queue_id < dev->data->nb_tx_queues) {
 711                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 712
 713                 /* reset head and tail pointers */
 714                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 715                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 716
 717                 /* enable TX queue */
 718                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 719                                         FM10K_TXDCTL_ENABLE | txdctl);
 720                 FM10K_WRITE_FLUSH(hw);
 721         } else
 722                 err = -1;
 723
 724         return err;
 725 }
 726
 727 static int
 728 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 729 {
 730         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 731
 732         PMD_INIT_FUNC_TRACE();
 733
 734         if (tx_queue_id < dev->data->nb_tx_queues) {
 735                 tx_queue_disable(hw, tx_queue_id);
 736                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 737         }
 738
 739         return 0;
 740 }
 741
 742 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 743 {
 744         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 745                 != FM10K_DGLORTMAP_NONE);
 746 }
 747
 748 static void
 749 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 750 {
 751         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 752         int status;
 753
 754         PMD_INIT_FUNC_TRACE();
 755
 756         /* Return if it didn't acquire valid glort range */
 757         if (!fm10k_glort_valid(hw))
 758                 return;
 759
 760         fm10k_mbx_lock(hw);
 761         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 762                                 FM10K_XCAST_MODE_PROMISC);
 763         fm10k_mbx_unlock(hw);
 764
 765         if (status != FM10K_SUCCESS)
 766                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 767 }
 768
 769 static void
 770 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 771 {
 772         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 773         uint8_t mode;
 774         int status;
 775
 776         PMD_INIT_FUNC_TRACE();
 777
 778         /* Return if it didn't acquire valid glort range */
 779         if (!fm10k_glort_valid(hw))
 780                 return;
 781
 782         if (dev->data->all_multicast == 1)
 783                 mode = FM10K_XCAST_MODE_ALLMULTI;
 784         else
 785                 mode = FM10K_XCAST_MODE_NONE;
 786
 787         fm10k_mbx_lock(hw);
 788         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 789                                 mode);
 790         fm10k_mbx_unlock(hw);
 791
 792         if (status != FM10K_SUCCESS)
 793                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 794 }
 795
 796 static void
 797 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 798 {
 799         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 800         int status;
 801
 802         PMD_INIT_FUNC_TRACE();
 803
 804         /* Return if it didn't acquire valid glort range */
 805         if (!fm10k_glort_valid(hw))
 806                 return;
 807
 808         /* If promiscuous mode is enabled, it doesn't make sense to enable
 809          * allmulticast and disable promiscuous since fm10k only can select
 810          * one of the modes.
 811          */
 812         if (dev->data->promiscuous) {
 813                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 814                         "needn't enable allmulticast");
 815                 return;
 816         }
 817
 818         fm10k_mbx_lock(hw);
 819         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 820                                 FM10K_XCAST_MODE_ALLMULTI);
 821         fm10k_mbx_unlock(hw);
 822
 823         if (status != FM10K_SUCCESS)
 824                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 825 }
 826
 827 static void
 828 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 829 {
 830         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 831         int status;
 832
 833         PMD_INIT_FUNC_TRACE();
 834
 835         /* Return if it didn't acquire valid glort range */
 836         if (!fm10k_glort_valid(hw))
 837                 return;
 838
 839         if (dev->data->promiscuous) {
 840                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 841                         "since promisc mode is enabled");
 842                 return;
 843         }
 844
 845         fm10k_mbx_lock(hw);
 846         /* Change mode to unicast mode */
 847         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 848                                 FM10K_XCAST_MODE_NONE);
 849         fm10k_mbx_unlock(hw);
 850
 851         if (status != FM10K_SUCCESS)
 852                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 853 }
 854
 855 static void
 856 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 857 {
 858         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 859         uint32_t dglortdec, pool_len, rss_len, i;
 860         uint16_t nb_queue_pools;
 861         struct fm10k_macvlan_filter_info *macvlan;
 862
 863         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 864         nb_queue_pools = macvlan->nb_queue_pools;
 865         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 866         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 867         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 868
 869         /* Establish only MAP 0 as valid */
 870         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 871
 872         /* Configure VMDQ/RSS DGlort Decoder */
 873         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 874
 875         /* Invalidate all other GLORT entries */
 876         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 877                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 878                                 FM10K_DGLORTMAP_NONE);
 879 }
 880
 881 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 882 static int
 883 fm10k_dev_start(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886         int i, diag;
 887
 888         PMD_INIT_FUNC_TRACE();
 889
 890         /* stop, init, then start the hw */
 891         diag = fm10k_stop_hw(hw);
 892         if (diag != FM10K_SUCCESS) {
 893                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 894                 return -EIO;
 895         }
 896
 897         diag = fm10k_init_hw(hw);
 898         if (diag != FM10K_SUCCESS) {
 899                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 900                 return -EIO;
 901         }
 902
 903         diag = fm10k_start_hw(hw);
 904         if (diag != FM10K_SUCCESS) {
 905                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 906                 return -EIO;
 907         }
 908
 909         diag = fm10k_dev_tx_init(dev);
 910         if (diag) {
 911                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 912                 return diag;
 913         }
 914
 915         diag = fm10k_dev_rx_init(dev);
 916         if (diag) {
 917                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 918                 return diag;
 919         }
 920
 921         if (hw->mac.type == fm10k_mac_pf)
 922                 fm10k_dev_dglort_map_configure(dev);
 923
 924         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 925                 struct fm10k_rx_queue *rxq;
 926                 rxq = dev->data->rx_queues[i];
 927
 928                 if (rxq->rx_deferred_start)
 929                         continue;
 930                 diag = fm10k_dev_rx_queue_start(dev, i);
 931                 if (diag != 0) {
 932                         int j;
 933                         for (j = 0; j < i; ++j)
 934                                 rx_queue_clean(dev->data->rx_queues[j]);
 935                         return diag;
 936                 }
 937         }
 938
 939         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 940                 struct fm10k_tx_queue *txq;
 941                 txq = dev->data->tx_queues[i];
 942
 943                 if (txq->tx_deferred_start)
 944                         continue;
 945                 diag = fm10k_dev_tx_queue_start(dev, i);
 946                 if (diag != 0) {
 947                         int j;
 948                         for (j = 0; j < i; ++j)
 949                                 tx_queue_clean(dev->data->tx_queues[j]);
 950                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 951                                 rx_queue_clean(dev->data->rx_queues[j]);
 952                         return diag;
 953                 }
 954         }
 955
 956         /* Update default vlan when not in VMDQ mode */
 957         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 958                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 959
 960         return 0;
 961 }
 962
 963 static void
 964 fm10k_dev_stop(struct rte_eth_dev *dev)
 965 {
 966         int i;
 967
 968         PMD_INIT_FUNC_TRACE();
 969
 970         if (dev->data->tx_queues)
 971                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 972                         fm10k_dev_tx_queue_stop(dev, i);
 973
 974         if (dev->data->rx_queues)
 975                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 976                         fm10k_dev_rx_queue_stop(dev, i);
 977 }
 978
 979 static void
 980 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 981 {
 982         int i;
 983
 984         PMD_INIT_FUNC_TRACE();
 985
 986         if (dev->data->tx_queues) {
 987                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 988                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 989         }
 990
 991         if (dev->data->rx_queues) {
 992                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 993                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 994         }
 995 }
 996
 997 static void
 998 fm10k_dev_close(struct rte_eth_dev *dev)
 999 {
1000         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1001         uint16_t nb_lport;
1002         struct fm10k_macvlan_filter_info *macvlan;
1003
1004         PMD_INIT_FUNC_TRACE();
1005
1006         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1007         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1008         fm10k_mbx_lock(hw);
1009         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1010                 nb_lport, false);
1011         fm10k_mbx_unlock(hw);
1012
1013         /* Stop mailbox service first */
1014         fm10k_close_mbx_service(hw);
1015         fm10k_dev_stop(dev);
1016         fm10k_dev_queue_release(dev);
1017         fm10k_stop_hw(hw);
1018 }
1019
1020 static int
1021 fm10k_link_update(struct rte_eth_dev *dev,
1022         __rte_unused int wait_to_complete)
1023 {
1024         PMD_INIT_FUNC_TRACE();
1025
1026         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1027          * x8 PCIe interface. For now, we leave the speed undefined since there
1028          * is no 50Gbps Ethernet. */
1029         dev->data->dev_link.link_speed  = 0;
1030         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1031         dev->data->dev_link.link_status = 1;
1032
1033         return 0;
1034 }
1035
1036 static void
1037 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1038 {
1039         uint64_t ipackets, opackets, ibytes, obytes;
1040         struct fm10k_hw *hw =
1041                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1042         struct fm10k_hw_stats *hw_stats =
1043                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1044         int i;
1045
1046         PMD_INIT_FUNC_TRACE();
1047
1048         fm10k_update_hw_stats(hw, hw_stats);
1049
1050         ipackets = opackets = ibytes = obytes = 0;
1051         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1052                 (i < hw->mac.max_queues); ++i) {
1053                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1054                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1055                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1056                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1057                 ipackets += stats->q_ipackets[i];
1058                 opackets += stats->q_opackets[i];
1059                 ibytes   += stats->q_ibytes[i];
1060                 obytes   += stats->q_obytes[i];
1061         }
1062         stats->ipackets = ipackets;
1063         stats->opackets = opackets;
1064         stats->ibytes = ibytes;
1065         stats->obytes = obytes;
1066 }
1067
1068 static void
1069 fm10k_stats_reset(struct rte_eth_dev *dev)
1070 {
1071         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1072         struct fm10k_hw_stats *hw_stats =
1073                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1074
1075         PMD_INIT_FUNC_TRACE();
1076
1077         memset(hw_stats, 0, sizeof(*hw_stats));
1078         fm10k_rebind_hw_stats(hw, hw_stats);
1079 }
1080
1081 static void
1082 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1083         struct rte_eth_dev_info *dev_info)
1084 {
1085         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1086
1087         PMD_INIT_FUNC_TRACE();
1088
1089         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1090         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1091         dev_info->max_rx_queues      = hw->mac.max_queues;
1092         dev_info->max_tx_queues      = hw->mac.max_queues;
1093         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1094         dev_info->max_hash_mac_addrs = 0;
1095         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1096         dev_info->vmdq_pool_base     = 0;
1097         dev_info->vmdq_queue_base    = 0;
1098         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1099         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1100         dev_info->rx_offload_capa =
1101                 DEV_RX_OFFLOAD_VLAN_STRIP |
1102                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1103                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1104                 DEV_RX_OFFLOAD_TCP_CKSUM;
1105         dev_info->tx_offload_capa =
1106                 DEV_TX_OFFLOAD_VLAN_INSERT |
1107                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1108                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1109                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1110                 DEV_TX_OFFLOAD_TCP_TSO;
1111
1112         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1113         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1114
1115         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1116                 .rx_thresh = {
1117                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1118                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1119                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1120                 },
1121                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1122                 .rx_drop_en = 0,
1123         };
1124
1125         dev_info->default_txconf = (struct rte_eth_txconf) {
1126                 .tx_thresh = {
1127                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1128                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1129                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1130                 },
1131                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1132                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1133                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1134                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1135         };
1136
1137 }
1138
1139 static int
1140 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1141 {
1142         s32 result;
1143         uint16_t mac_num = 0;
1144         uint32_t vid_idx, vid_bit, mac_index;
1145         struct fm10k_hw *hw;
1146         struct fm10k_macvlan_filter_info *macvlan;
1147         struct rte_eth_dev_data *data = dev->data;
1148
1149         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1150         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1151
1152         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1153                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1154                 return (-EINVAL);
1155         }
1156
1157         if (vlan_id > ETH_VLAN_ID_MAX) {
1158                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1159                 return (-EINVAL);
1160         }
1161
1162         vid_idx = FM10K_VFTA_IDX(vlan_id);
1163         vid_bit = FM10K_VFTA_BIT(vlan_id);
1164         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1165         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1166                 return 0;
1167         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1168         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1169                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1170                         "in the VLAN filter table");
1171                 return (-EINVAL);
1172         }
1173
1174         fm10k_mbx_lock(hw);
1175         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1176         fm10k_mbx_unlock(hw);
1177         if (result != FM10K_SUCCESS) {
1178                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1179                 return (-EIO);
1180         }
1181
1182         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1183                         (result == FM10K_SUCCESS); mac_index++) {
1184                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1185                         continue;
1186                 if (mac_num > macvlan->mac_num - 1) {
1187                         PMD_INIT_LOG(ERR, "MAC address number "
1188                                         "not match");
1189                         break;
1190                 }
1191                 fm10k_mbx_lock(hw);
1192                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1193                         data->mac_addrs[mac_index].addr_bytes,
1194                         vlan_id, on, 0);
1195                 fm10k_mbx_unlock(hw);
1196                 mac_num++;
1197         }
1198         if (result != FM10K_SUCCESS) {
1199                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1200                 return (-EIO);
1201         }
1202
1203         if (on) {
1204                 macvlan->vlan_num++;
1205                 macvlan->vfta[vid_idx] |= vid_bit;
1206         } else {
1207                 macvlan->vlan_num--;
1208                 macvlan->vfta[vid_idx] &= ~vid_bit;
1209         }
1210         return 0;
1211 }
1212
1213 static void
1214 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1215 {
1216         if (mask & ETH_VLAN_STRIP_MASK) {
1217                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1218                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1219                                         "always on in fm10k");
1220         }
1221
1222         if (mask & ETH_VLAN_EXTEND_MASK) {
1223                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1224                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1225                                         "supported in fm10k");
1226         }
1227
1228         if (mask & ETH_VLAN_FILTER_MASK) {
1229                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1230                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1231         }
1232 }
1233
1234 /* Add/Remove a MAC address, and update filters to main VSI */
1235 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1236                 const u8 *mac, bool add, uint32_t pool)
1237 {
1238         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1239         struct fm10k_macvlan_filter_info *macvlan;
1240         uint32_t i, j, k;
1241
1242         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1243
1244         if (pool != MAIN_VSI_POOL_NUMBER) {
1245                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1246                         "mac to pool %u", pool);
1247                 return;
1248         }
1249         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1250                 if (!macvlan->vfta[j])
1251                         continue;
1252                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1253                         if (!(macvlan->vfta[j] & (1 << k)))
1254                                 continue;
1255                         if (i + 1 > macvlan->vlan_num) {
1256                                 PMD_INIT_LOG(ERR, "vlan number not match");
1257                                 return;
1258                         }
1259                         fm10k_mbx_lock(hw);
1260                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1261                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1262                         fm10k_mbx_unlock(hw);
1263                         i++;
1264                 }
1265         }
1266 }
1267
1268 /* Add/Remove a MAC address, and update filters to VMDQ */
1269 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1270                 const u8 *mac, bool add, uint32_t pool)
1271 {
1272         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1273         struct fm10k_macvlan_filter_info *macvlan;
1274         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1275         uint32_t i;
1276
1277         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1278         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1279
1280         if (pool > macvlan->nb_queue_pools) {
1281                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1282                         " Max pool is %u",
1283                         pool, macvlan->nb_queue_pools);
1284                 return;
1285         }
1286         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1287                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1288                         continue;
1289                 fm10k_mbx_lock(hw);
1290                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1291                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1292                 fm10k_mbx_unlock(hw);
1293         }
1294 }
1295
1296 /* Add/Remove a MAC address, and update filters */
1297 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1298                 const u8 *mac, bool add, uint32_t pool)
1299 {
1300         struct fm10k_macvlan_filter_info *macvlan;
1301
1302         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1303
1304         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1305                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1306         else
1307                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1308
1309         if (add)
1310                 macvlan->mac_num++;
1311         else
1312                 macvlan->mac_num--;
1313 }
1314
1315 /* Add a MAC address, and update filters */
1316 static void
1317 fm10k_macaddr_add(struct rte_eth_dev *dev,
1318                 struct ether_addr *mac_addr,
1319                 uint32_t index,
1320                 uint32_t pool)
1321 {
1322         struct fm10k_macvlan_filter_info *macvlan;
1323
1324         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1325         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1326         macvlan->mac_vmdq_id[index] = pool;
1327 }
1328
1329 /* Remove a MAC address, and update filters */
1330 static void
1331 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1332 {
1333         struct rte_eth_dev_data *data = dev->data;
1334         struct fm10k_macvlan_filter_info *macvlan;
1335
1336         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1337         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1338                         FALSE, macvlan->mac_vmdq_id[index]);
1339         macvlan->mac_vmdq_id[index] = 0;
1340 }
1341
1342 static inline int
1343 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1344 {
1345         if ((request < min) || (request > max) || ((request % mult) != 0))
1346                 return -1;
1347         else
1348                 return 0;
1349 }
1350
1351 /*
1352  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1353  * the physical address is required. If the memzone is already created, then
1354  * this function returns a pointer to the existing memzone.
1355  */
1356 static inline const struct rte_memzone *
1357 allocate_hw_ring(const char *driver_name, const char *ring_name,
1358         uint8_t port_id, uint16_t queue_id, int socket_id,
1359         uint32_t size, uint32_t align)
1360 {
1361         char name[RTE_MEMZONE_NAMESIZE];
1362         const struct rte_memzone *mz;
1363
1364         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1365                  driver_name, ring_name, port_id, queue_id, socket_id);
1366
1367         /* return the memzone if it already exists */
1368         mz = rte_memzone_lookup(name);
1369         if (mz)
1370                 return mz;
1371
1372 #ifdef RTE_LIBRTE_XEN_DOM0
1373         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1374                                            RTE_PGSIZE_2M);
1375 #else
1376         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1377 #endif
1378 }
1379
1380 static inline int
1381 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1382 {
1383         if ((request < min) || (request > max) || ((div % request) != 0))
1384                 return -1;
1385         else
1386                 return 0;
1387 }
1388
1389 static inline int
1390 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1391 {
1392         uint16_t rx_free_thresh;
1393
1394         if (conf->rx_free_thresh == 0)
1395                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1396         else
1397                 rx_free_thresh = conf->rx_free_thresh;
1398
1399         /* make sure the requested threshold satisfies the constraints */
1400         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1401                         FM10K_RX_FREE_THRESH_MAX(q),
1402                         FM10K_RX_FREE_THRESH_DIV(q),
1403                         rx_free_thresh)) {
1404                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1405                         "less than or equal to %u, "
1406                         "greater than or equal to %u, "
1407                         "and a divisor of %u",
1408                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1409                         FM10K_RX_FREE_THRESH_MIN(q),
1410                         FM10K_RX_FREE_THRESH_DIV(q));
1411                 return (-EINVAL);
1412         }
1413
1414         q->alloc_thresh = rx_free_thresh;
1415         q->drop_en = conf->rx_drop_en;
1416         q->rx_deferred_start = conf->rx_deferred_start;
1417
1418         return 0;
1419 }
1420
1421 /*
1422  * Hardware requires specific alignment for Rx packet buffers. At
1423  * least one of the following two conditions must be satisfied.
1424  *  1. Address is 512B aligned
1425  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1426  *
1427  * As such, the driver may need to adjust the DMA address within the
1428  * buffer by up to 512B.
1429  *
1430  * return 1 if the element size is valid, otherwise return 0.
1431  */
1432 static int
1433 mempool_element_size_valid(struct rte_mempool *mp)
1434 {
1435         uint32_t min_size;
1436
1437         /* elt_size includes mbuf header and headroom */
1438         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1439                         RTE_PKTMBUF_HEADROOM;
1440
1441         /* account for up to 512B of alignment */
1442         min_size -= FM10K_RX_DATABUF_ALIGN;
1443
1444         /* sanity check for overflow */
1445         if (min_size > mp->elt_size)
1446                 return 0;
1447
1448         /* size is valid */
1449         return 1;
1450 }
1451
1452 static int
1453 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1454         uint16_t nb_desc, unsigned int socket_id,
1455         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1456 {
1457         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1458         struct fm10k_rx_queue *q;
1459         const struct rte_memzone *mz;
1460
1461         PMD_INIT_FUNC_TRACE();
1462
1463         /* make sure the mempool element size can account for alignment. */
1464         if (!mempool_element_size_valid(mp)) {
1465                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1466                 return (-EINVAL);
1467         }
1468
1469         /* make sure a valid number of descriptors have been requested */
1470         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1471                                 FM10K_MULT_RX_DESC, nb_desc)) {
1472                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1473                         "less than or equal to %"PRIu32", "
1474                         "greater than or equal to %u, "
1475                         "and a multiple of %u",
1476                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1477                         FM10K_MULT_RX_DESC);
1478                 return (-EINVAL);
1479         }
1480
1481         /*
1482          * if this queue existed already, free the associated memory. The
1483          * queue cannot be reused in case we need to allocate memory on
1484          * different socket than was previously used.
1485          */
1486         if (dev->data->rx_queues[queue_id] != NULL) {
1487                 rx_queue_free(dev->data->rx_queues[queue_id]);
1488                 dev->data->rx_queues[queue_id] = NULL;
1489         }
1490
1491         /* allocate memory for the queue structure */
1492         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1493                                 socket_id);
1494         if (q == NULL) {
1495                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1496                 return (-ENOMEM);
1497         }
1498
1499         /* setup queue */
1500         q->mp = mp;
1501         q->nb_desc = nb_desc;
1502         q->port_id = dev->data->port_id;
1503         q->queue_id = queue_id;
1504         q->tail_ptr = (volatile uint32_t *)
1505                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1506         if (handle_rxconf(q, conf))
1507                 return (-EINVAL);
1508
1509         /* allocate memory for the software ring */
1510         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1511                                         nb_desc * sizeof(struct rte_mbuf *),
1512                                         RTE_CACHE_LINE_SIZE, socket_id);
1513         if (q->sw_ring == NULL) {
1514                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1515                 rte_free(q);
1516                 return (-ENOMEM);
1517         }
1518
1519         /*
1520          * allocate memory for the hardware descriptor ring. A memzone large
1521          * enough to hold the maximum ring size is requested to allow for
1522          * resizing in later calls to the queue setup function.
1523          */
1524         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1525                                 dev->data->port_id, queue_id, socket_id,
1526                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1527         if (mz == NULL) {
1528                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1529                 rte_free(q->sw_ring);
1530                 rte_free(q);
1531                 return (-ENOMEM);
1532         }
1533         q->hw_ring = mz->addr;
1534 #ifdef RTE_LIBRTE_XEN_DOM0
1535         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1536 #else
1537         q->hw_ring_phys_addr = mz->phys_addr;
1538 #endif
1539
1540         dev->data->rx_queues[queue_id] = q;
1541         return 0;
1542 }
1543
1544 static void
1545 fm10k_rx_queue_release(void *queue)
1546 {
1547         PMD_INIT_FUNC_TRACE();
1548
1549         rx_queue_free(queue);
1550 }
1551
1552 static inline int
1553 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1554 {
1555         uint16_t tx_free_thresh;
1556         uint16_t tx_rs_thresh;
1557
1558         /* constraint MACROs require that tx_free_thresh is configured
1559          * before tx_rs_thresh */
1560         if (conf->tx_free_thresh == 0)
1561                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1562         else
1563                 tx_free_thresh = conf->tx_free_thresh;
1564
1565         /* make sure the requested threshold satisfies the constraints */
1566         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1567                         FM10K_TX_FREE_THRESH_MAX(q),
1568                         FM10K_TX_FREE_THRESH_DIV(q),
1569                         tx_free_thresh)) {
1570                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1571                         "less than or equal to %u, "
1572                         "greater than or equal to %u, "
1573                         "and a divisor of %u",
1574                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1575                         FM10K_TX_FREE_THRESH_MIN(q),
1576                         FM10K_TX_FREE_THRESH_DIV(q));
1577                 return (-EINVAL);
1578         }
1579
1580         q->free_thresh = tx_free_thresh;
1581
1582         if (conf->tx_rs_thresh == 0)
1583                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1584         else
1585                 tx_rs_thresh = conf->tx_rs_thresh;
1586
1587         q->tx_deferred_start = conf->tx_deferred_start;
1588
1589         /* make sure the requested threshold satisfies the constraints */
1590         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1591                         FM10K_TX_RS_THRESH_MAX(q),
1592                         FM10K_TX_RS_THRESH_DIV(q),
1593                         tx_rs_thresh)) {
1594                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1595                         "less than or equal to %u, "
1596                         "greater than or equal to %u, "
1597                         "and a divisor of %u",
1598                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1599                         FM10K_TX_RS_THRESH_MIN(q),
1600                         FM10K_TX_RS_THRESH_DIV(q));
1601                 return (-EINVAL);
1602         }
1603
1604         q->rs_thresh = tx_rs_thresh;
1605
1606         return 0;
1607 }
1608
1609 static int
1610 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1611         uint16_t nb_desc, unsigned int socket_id,
1612         const struct rte_eth_txconf *conf)
1613 {
1614         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1615         struct fm10k_tx_queue *q;
1616         const struct rte_memzone *mz;
1617
1618         PMD_INIT_FUNC_TRACE();
1619
1620         /* make sure a valid number of descriptors have been requested */
1621         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1622                                 FM10K_MULT_TX_DESC, nb_desc)) {
1623                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1624                         "less than or equal to %"PRIu32", "
1625                         "greater than or equal to %u, "
1626                         "and a multiple of %u",
1627                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1628                         FM10K_MULT_TX_DESC);
1629                 return (-EINVAL);
1630         }
1631
1632         /*
1633          * if this queue existed already, free the associated memory. The
1634          * queue cannot be reused in case we need to allocate memory on
1635          * different socket than was previously used.
1636          */
1637         if (dev->data->tx_queues[queue_id] != NULL) {
1638                 tx_queue_free(dev->data->tx_queues[queue_id]);
1639                 dev->data->tx_queues[queue_id] = NULL;
1640         }
1641
1642         /* allocate memory for the queue structure */
1643         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1644                                 socket_id);
1645         if (q == NULL) {
1646                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1647                 return (-ENOMEM);
1648         }
1649
1650         /* setup queue */
1651         q->nb_desc = nb_desc;
1652         q->port_id = dev->data->port_id;
1653         q->queue_id = queue_id;
1654         q->tail_ptr = (volatile uint32_t *)
1655                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1656         if (handle_txconf(q, conf))
1657                 return (-EINVAL);
1658
1659         /* allocate memory for the software ring */
1660         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1661                                         nb_desc * sizeof(struct rte_mbuf *),
1662                                         RTE_CACHE_LINE_SIZE, socket_id);
1663         if (q->sw_ring == NULL) {
1664                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1665                 rte_free(q);
1666                 return (-ENOMEM);
1667         }
1668
1669         /*
1670          * allocate memory for the hardware descriptor ring. A memzone large
1671          * enough to hold the maximum ring size is requested to allow for
1672          * resizing in later calls to the queue setup function.
1673          */
1674         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1675                                 dev->data->port_id, queue_id, socket_id,
1676                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1677         if (mz == NULL) {
1678                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1679                 rte_free(q->sw_ring);
1680                 rte_free(q);
1681                 return (-ENOMEM);
1682         }
1683         q->hw_ring = mz->addr;
1684 #ifdef RTE_LIBRTE_XEN_DOM0
1685         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1686 #else
1687         q->hw_ring_phys_addr = mz->phys_addr;
1688 #endif
1689
1690         /*
1691          * allocate memory for the RS bit tracker. Enough slots to hold the
1692          * descriptor index for each RS bit needing to be set are required.
1693          */
1694         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1695                                 ((nb_desc + 1) / q->rs_thresh) *
1696                                 sizeof(uint16_t),
1697                                 RTE_CACHE_LINE_SIZE, socket_id);
1698         if (q->rs_tracker.list == NULL) {
1699                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1700                 rte_free(q->sw_ring);
1701                 rte_free(q);
1702                 return (-ENOMEM);
1703         }
1704
1705         dev->data->tx_queues[queue_id] = q;
1706         return 0;
1707 }
1708
1709 static void
1710 fm10k_tx_queue_release(void *queue)
1711 {
1712         PMD_INIT_FUNC_TRACE();
1713
1714         tx_queue_free(queue);
1715 }
1716
1717 static int
1718 fm10k_reta_update(struct rte_eth_dev *dev,
1719                         struct rte_eth_rss_reta_entry64 *reta_conf,
1720                         uint16_t reta_size)
1721 {
1722         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1723         uint16_t i, j, idx, shift;
1724         uint8_t mask;
1725         uint32_t reta;
1726
1727         PMD_INIT_FUNC_TRACE();
1728
1729         if (reta_size > FM10K_MAX_RSS_INDICES) {
1730                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1731                         "(%d) doesn't match the number hardware can supported "
1732                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1733                 return -EINVAL;
1734         }
1735
1736         /*
1737          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1738          * 128-entries in 32 registers
1739          */
1740         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1741                 idx = i / RTE_RETA_GROUP_SIZE;
1742                 shift = i % RTE_RETA_GROUP_SIZE;
1743                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1744                                 BIT_MASK_PER_UINT32);
1745                 if (mask == 0)
1746                         continue;
1747
1748                 reta = 0;
1749                 if (mask != BIT_MASK_PER_UINT32)
1750                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1751
1752                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1753                         if (mask & (0x1 << j)) {
1754                                 if (mask != 0xF)
1755                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1756                                 reta |= reta_conf[idx].reta[shift + j] <<
1757                                                 (CHAR_BIT * j);
1758                         }
1759                 }
1760                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1761         }
1762
1763         return 0;
1764 }
1765
1766 static int
1767 fm10k_reta_query(struct rte_eth_dev *dev,
1768                         struct rte_eth_rss_reta_entry64 *reta_conf,
1769                         uint16_t reta_size)
1770 {
1771         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1772         uint16_t i, j, idx, shift;
1773         uint8_t mask;
1774         uint32_t reta;
1775
1776         PMD_INIT_FUNC_TRACE();
1777
1778         if (reta_size < FM10K_MAX_RSS_INDICES) {
1779                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1780                         "(%d) doesn't match the number hardware can supported "
1781                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1782                 return -EINVAL;
1783         }
1784
1785         /*
1786          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1787          * 128-entries in 32 registers
1788          */
1789         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1790                 idx = i / RTE_RETA_GROUP_SIZE;
1791                 shift = i % RTE_RETA_GROUP_SIZE;
1792                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1793                                 BIT_MASK_PER_UINT32);
1794                 if (mask == 0)
1795                         continue;
1796
1797                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1798                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1799                         if (mask & (0x1 << j))
1800                                 reta_conf[idx].reta[shift + j] = ((reta >>
1801                                         CHAR_BIT * j) & UINT8_MAX);
1802                 }
1803         }
1804
1805         return 0;
1806 }
1807
1808 static int
1809 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1810         struct rte_eth_rss_conf *rss_conf)
1811 {
1812         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1813         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1814         uint32_t mrqc;
1815         uint64_t hf = rss_conf->rss_hf;
1816         int i;
1817
1818         PMD_INIT_FUNC_TRACE();
1819
1820         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1821                 FM10K_RSSRK_ENTRIES_PER_REG)
1822                 return -EINVAL;
1823
1824         if (hf == 0)
1825                 return -EINVAL;
1826
1827         mrqc = 0;
1828         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1829         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1830         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1831         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1832         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1833         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1834         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1835         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1836         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1837
1838         /* If the mapping doesn't fit any supported, return */
1839         if (mrqc == 0)
1840                 return -EINVAL;
1841
1842         if (key != NULL)
1843                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1844                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1845
1846         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1847
1848         return 0;
1849 }
1850
1851 static int
1852 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1853         struct rte_eth_rss_conf *rss_conf)
1854 {
1855         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1856         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1857         uint32_t mrqc;
1858         uint64_t hf;
1859         int i;
1860
1861         PMD_INIT_FUNC_TRACE();
1862
1863         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1864                                 FM10K_RSSRK_ENTRIES_PER_REG)
1865                 return -EINVAL;
1866
1867         if (key != NULL)
1868                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1869                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1870
1871         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1872         hf = 0;
1873         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1874         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1875         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1876         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1877         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1878         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1879         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1880         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1881         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1882
1883         rss_conf->rss_hf = hf;
1884
1885         return 0;
1886 }
1887
1888 static void
1889 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1890 {
1891         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1892         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1893
1894         /* Bind all local non-queue interrupt to vector 0 */
1895         int_map |= 0;
1896
1897         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1898         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1899         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1900         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1901         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1902         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1903
1904         /* Enable misc causes */
1905         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1906                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1907                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1908                                 FM10K_EIMR_ENABLE(MAILBOX) |
1909                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1910                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1911                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1912                                 FM10K_EIMR_ENABLE(VFLR));
1913
1914         /* Enable ITR 0 */
1915         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1916                                         FM10K_ITR_MASK_CLEAR);
1917         FM10K_WRITE_FLUSH(hw);
1918 }
1919
1920 static void
1921 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1922 {
1923         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1924         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1925
1926         int_map |= 0;
1927
1928         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1929         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1930         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1931         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1932         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1933         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1934
1935         /* Disable misc causes */
1936         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1937                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1938                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1939                                 FM10K_EIMR_DISABLE(MAILBOX) |
1940                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1941                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1942                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1943                                 FM10K_EIMR_DISABLE(VFLR));
1944
1945         /* Disable ITR 0 */
1946         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1947         FM10K_WRITE_FLUSH(hw);
1948 }
1949
1950 static void
1951 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1952 {
1953         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1954         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1955
1956         /* Bind all local non-queue interrupt to vector 0 */
1957         int_map |= 0;
1958
1959         /* Only INT 0 available, other 15 are reserved. */
1960         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1961
1962         /* Enable ITR 0 */
1963         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1964                                         FM10K_ITR_MASK_CLEAR);
1965         FM10K_WRITE_FLUSH(hw);
1966 }
1967
1968 static void
1969 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1970 {
1971         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1972         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1973
1974         int_map |= 0;
1975
1976         /* Only INT 0 available, other 15 are reserved. */
1977         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1978
1979         /* Disable ITR 0 */
1980         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1981         FM10K_WRITE_FLUSH(hw);
1982 }
1983
1984 static int
1985 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1986 {
1987         struct fm10k_fault fault;
1988         int err;
1989         const char *estr = "Unknown error";
1990
1991         /* Process PCA fault */
1992         if (eicr & FM10K_EICR_PCA_FAULT) {
1993                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1994                 if (err)
1995                         goto error;
1996                 switch (fault.type) {
1997                 case PCA_NO_FAULT:
1998                         estr = "PCA_NO_FAULT"; break;
1999                 case PCA_UNMAPPED_ADDR:
2000                         estr = "PCA_UNMAPPED_ADDR"; break;
2001                 case PCA_BAD_QACCESS_PF:
2002                         estr = "PCA_BAD_QACCESS_PF"; break;
2003                 case PCA_BAD_QACCESS_VF:
2004                         estr = "PCA_BAD_QACCESS_VF"; break;
2005                 case PCA_MALICIOUS_REQ:
2006                         estr = "PCA_MALICIOUS_REQ"; break;
2007                 case PCA_POISONED_TLP:
2008                         estr = "PCA_POISONED_TLP"; break;
2009                 case PCA_TLP_ABORT:
2010                         estr = "PCA_TLP_ABORT"; break;
2011                 default:
2012                         goto error;
2013                 }
2014                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2015                         estr, fault.func ? "VF" : "PF", fault.func,
2016                         fault.address, fault.specinfo);
2017         }
2018
2019         /* Process THI fault */
2020         if (eicr & FM10K_EICR_THI_FAULT) {
2021                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2022                 if (err)
2023                         goto error;
2024                 switch (fault.type) {
2025                 case THI_NO_FAULT:
2026                         estr = "THI_NO_FAULT"; break;
2027                 case THI_MAL_DIS_Q_FAULT:
2028                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2029                 default:
2030                         goto error;
2031                 }
2032                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2033                         estr, fault.func ? "VF" : "PF", fault.func,
2034                         fault.address, fault.specinfo);
2035         }
2036
2037         /* Process FUM fault */
2038         if (eicr & FM10K_EICR_FUM_FAULT) {
2039                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2040                 if (err)
2041                         goto error;
2042                 switch (fault.type) {
2043                 case FUM_NO_FAULT:
2044                         estr = "FUM_NO_FAULT"; break;
2045                 case FUM_UNMAPPED_ADDR:
2046                         estr = "FUM_UNMAPPED_ADDR"; break;
2047                 case FUM_POISONED_TLP:
2048                         estr = "FUM_POISONED_TLP"; break;
2049                 case FUM_BAD_VF_QACCESS:
2050                         estr = "FUM_BAD_VF_QACCESS"; break;
2051                 case FUM_ADD_DECODE_ERR:
2052                         estr = "FUM_ADD_DECODE_ERR"; break;
2053                 case FUM_RO_ERROR:
2054                         estr = "FUM_RO_ERROR"; break;
2055                 case FUM_QPRC_CRC_ERROR:
2056                         estr = "FUM_QPRC_CRC_ERROR"; break;
2057                 case FUM_CSR_TIMEOUT:
2058                         estr = "FUM_CSR_TIMEOUT"; break;
2059                 case FUM_INVALID_TYPE:
2060                         estr = "FUM_INVALID_TYPE"; break;
2061                 case FUM_INVALID_LENGTH:
2062                         estr = "FUM_INVALID_LENGTH"; break;
2063                 case FUM_INVALID_BE:
2064                         estr = "FUM_INVALID_BE"; break;
2065                 case FUM_INVALID_ALIGN:
2066                         estr = "FUM_INVALID_ALIGN"; break;
2067                 default:
2068                         goto error;
2069                 }
2070                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2071                         estr, fault.func ? "VF" : "PF", fault.func,
2072                         fault.address, fault.specinfo);
2073         }
2074
2075         return 0;
2076 error:
2077         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2078         return err;
2079 }
2080
2081 /**
2082  * PF interrupt handler triggered by NIC for handling specific interrupt.
2083  *
2084  * @param handle
2085  *  Pointer to interrupt handle.
2086  * @param param
2087  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2088  *
2089  * @return
2090  *  void
2091  */
2092 static void
2093 fm10k_dev_interrupt_handler_pf(
2094                         __rte_unused struct rte_intr_handle *handle,
2095                         void *param)
2096 {
2097         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2098         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2099         uint32_t cause, status;
2100
2101         if (hw->mac.type != fm10k_mac_pf)
2102                 return;
2103
2104         cause = FM10K_READ_REG(hw, FM10K_EICR);
2105
2106         /* Handle PCI fault cases */
2107         if (cause & FM10K_EICR_FAULT_MASK) {
2108                 PMD_INIT_LOG(ERR, "INT: find fault!");
2109                 fm10k_dev_handle_fault(hw, cause);
2110         }
2111
2112         /* Handle switch up/down */
2113         if (cause & FM10K_EICR_SWITCHNOTREADY)
2114                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2115
2116         if (cause & FM10K_EICR_SWITCHREADY)
2117                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2118
2119         /* Handle mailbox message */
2120         fm10k_mbx_lock(hw);
2121         hw->mbx.ops.process(hw, &hw->mbx);
2122         fm10k_mbx_unlock(hw);
2123
2124         /* Handle SRAM error */
2125         if (cause & FM10K_EICR_SRAMERROR) {
2126                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2127
2128                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2129                 /* Write to clear pending bits */
2130                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2131
2132                 /* Todo: print out error message after shared code  updates */
2133         }
2134
2135         /* Clear these 3 events if having any */
2136         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2137                  FM10K_EICR_SWITCHREADY;
2138         if (cause)
2139                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2140
2141         /* Re-enable interrupt from device side */
2142         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2143                                         FM10K_ITR_MASK_CLEAR);
2144         /* Re-enable interrupt from host side */
2145         rte_intr_enable(&(dev->pci_dev->intr_handle));
2146 }
2147
2148 /**
2149  * VF interrupt handler triggered by NIC for handling specific interrupt.
2150  *
2151  * @param handle
2152  *  Pointer to interrupt handle.
2153  * @param param
2154  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2155  *
2156  * @return
2157  *  void
2158  */
2159 static void
2160 fm10k_dev_interrupt_handler_vf(
2161                         __rte_unused struct rte_intr_handle *handle,
2162                         void *param)
2163 {
2164         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2165         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2166
2167         if (hw->mac.type != fm10k_mac_vf)
2168                 return;
2169
2170         /* Handle mailbox message if lock is acquired */
2171         fm10k_mbx_lock(hw);
2172         hw->mbx.ops.process(hw, &hw->mbx);
2173         fm10k_mbx_unlock(hw);
2174
2175         /* Re-enable interrupt from device side */
2176         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2177                                         FM10K_ITR_MASK_CLEAR);
2178         /* Re-enable interrupt from host side */
2179         rte_intr_enable(&(dev->pci_dev->intr_handle));
2180 }
2181
2182 /* Mailbox message handler in VF */
2183 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2184         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2185         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2186         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2187         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2188 };
2189
2190 /* Mailbox message handler in PF */
2191 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2192         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2193         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2194         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2195         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2196         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2197         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2198         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2199 };
2200
2201 static int
2202 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2203 {
2204         int err;
2205
2206         /* Initialize mailbox lock */
2207         fm10k_mbx_initlock(hw);
2208
2209         /* Replace default message handler with new ones */
2210         if (hw->mac.type == fm10k_mac_pf)
2211                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2212         else
2213                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2214
2215         if (err) {
2216                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2217                                 err);
2218                 return err;
2219         }
2220         /* Connect to SM for PF device or PF for VF device */
2221         return hw->mbx.ops.connect(hw, &hw->mbx);
2222 }
2223
2224 static void
2225 fm10k_close_mbx_service(struct fm10k_hw *hw)
2226 {
2227         /* Disconnect from SM for PF device or PF for VF device */
2228         hw->mbx.ops.disconnect(hw, &hw->mbx);
2229 }
2230
2231 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2232         .dev_configure          = fm10k_dev_configure,
2233         .dev_start              = fm10k_dev_start,
2234         .dev_stop               = fm10k_dev_stop,
2235         .dev_close              = fm10k_dev_close,
2236         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2237         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2238         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2239         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2240         .stats_get              = fm10k_stats_get,
2241         .stats_reset            = fm10k_stats_reset,
2242         .link_update            = fm10k_link_update,
2243         .dev_infos_get          = fm10k_dev_infos_get,
2244         .vlan_filter_set        = fm10k_vlan_filter_set,
2245         .vlan_offload_set       = fm10k_vlan_offload_set,
2246         .mac_addr_add           = fm10k_macaddr_add,
2247         .mac_addr_remove        = fm10k_macaddr_remove,
2248         .rx_queue_start         = fm10k_dev_rx_queue_start,
2249         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2250         .tx_queue_start         = fm10k_dev_tx_queue_start,
2251         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2252         .rx_queue_setup         = fm10k_rx_queue_setup,
2253         .rx_queue_release       = fm10k_rx_queue_release,
2254         .tx_queue_setup         = fm10k_tx_queue_setup,
2255         .tx_queue_release       = fm10k_tx_queue_release,
2256         .reta_update            = fm10k_reta_update,
2257         .reta_query             = fm10k_reta_query,
2258         .rss_hash_update        = fm10k_rss_hash_update,
2259         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2260 };
2261
2262 static int
2263 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2264 {
2265         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2266         int diag;
2267         struct fm10k_macvlan_filter_info *macvlan;
2268
2269         PMD_INIT_FUNC_TRACE();
2270
2271         dev->dev_ops = &fm10k_eth_dev_ops;
2272         dev->rx_pkt_burst = &fm10k_recv_pkts;
2273         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2274
2275         if (dev->data->scattered_rx)
2276                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2277
2278         /* only initialize in the primary process */
2279         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2280                 return 0;
2281
2282         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2283         memset(macvlan, 0, sizeof(*macvlan));
2284         /* Vendor and Device ID need to be set before init of shared code */
2285         memset(hw, 0, sizeof(*hw));
2286         hw->device_id = dev->pci_dev->id.device_id;
2287         hw->vendor_id = dev->pci_dev->id.vendor_id;
2288         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2289         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2290         hw->revision_id = 0;
2291         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2292         if (hw->hw_addr == NULL) {
2293                 PMD_INIT_LOG(ERR, "Bad mem resource."
2294                         " Try to blacklist unused devices.");
2295                 return -EIO;
2296         }
2297
2298         /* Store fm10k_adapter pointer */
2299         hw->back = dev->data->dev_private;
2300
2301         /* Initialize the shared code */
2302         diag = fm10k_init_shared_code(hw);
2303         if (diag != FM10K_SUCCESS) {
2304                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2305                 return -EIO;
2306         }
2307
2308         /*
2309          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2310          * there is no way to get link status without reading BAR4.  Until this
2311          * works, assume we have maximum bandwidth.
2312          * @todo - fix bus info
2313          */
2314         hw->bus_caps.speed = fm10k_bus_speed_8000;
2315         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2316         hw->bus_caps.payload = fm10k_bus_payload_512;
2317         hw->bus.speed = fm10k_bus_speed_8000;
2318         hw->bus.width = fm10k_bus_width_pcie_x8;
2319         hw->bus.payload = fm10k_bus_payload_256;
2320
2321         /* Initialize the hw */
2322         diag = fm10k_init_hw(hw);
2323         if (diag != FM10K_SUCCESS) {
2324                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2325                 return -EIO;
2326         }
2327
2328         /* Initialize MAC address(es) */
2329         dev->data->mac_addrs = rte_zmalloc("fm10k",
2330                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2331         if (dev->data->mac_addrs == NULL) {
2332                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2333                 return -ENOMEM;
2334         }
2335
2336         diag = fm10k_read_mac_addr(hw);
2337
2338         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2339                         &dev->data->mac_addrs[0]);
2340
2341         if (diag != FM10K_SUCCESS ||
2342                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2343
2344                 /* Generate a random addr */
2345                 eth_random_addr(hw->mac.addr);
2346                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2347                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2348                 &dev->data->mac_addrs[0]);
2349         }
2350
2351         /* Reset the hw statistics */
2352         fm10k_stats_reset(dev);
2353
2354         /* Reset the hw */
2355         diag = fm10k_reset_hw(hw);
2356         if (diag != FM10K_SUCCESS) {
2357                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2358                 return -EIO;
2359         }
2360
2361         /* Setup mailbox service */
2362         diag = fm10k_setup_mbx_service(hw);
2363         if (diag != FM10K_SUCCESS) {
2364                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2365                 return -EIO;
2366         }
2367
2368         /*PF/VF has different interrupt handling mechanism */
2369         if (hw->mac.type == fm10k_mac_pf) {
2370                 /* register callback func to eal lib */
2371                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2372                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2373
2374                 /* enable MISC interrupt */
2375                 fm10k_dev_enable_intr_pf(dev);
2376         } else { /* VF */
2377                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2378                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2379
2380                 fm10k_dev_enable_intr_vf(dev);
2381         }
2382
2383         /* Enable uio intr after callback registered */
2384         rte_intr_enable(&(dev->pci_dev->intr_handle));
2385
2386         hw->mac.ops.update_int_moderator(hw);
2387
2388         /* Make sure Switch Manager is ready before going forward. */
2389         if (hw->mac.type == fm10k_mac_pf) {
2390                 int switch_ready = 0;
2391                 int i;
2392
2393                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2394                         fm10k_mbx_lock(hw);
2395                         hw->mac.ops.get_host_state(hw, &switch_ready);
2396                         fm10k_mbx_unlock(hw);
2397                         if (switch_ready)
2398                                 break;
2399                         /* Delay some time to acquire async LPORT_MAP info. */
2400                         rte_delay_us(WAIT_SWITCH_MSG_US);
2401                 }
2402
2403                 if (switch_ready == 0) {
2404                         PMD_INIT_LOG(ERR, "switch is not ready");
2405                         return -1;
2406                 }
2407         }
2408
2409         /*
2410          * Below function will trigger operations on mailbox, acquire lock to
2411          * avoid race condition from interrupt handler. Operations on mailbox
2412          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2413          * will handle and generate an interrupt to our side. Then,  FIFO in
2414          * mailbox will be touched.
2415          */
2416         fm10k_mbx_lock(hw);
2417         /* Enable port first */
2418         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2419
2420         /* Set unicast mode by default. App can change to other mode in other
2421          * API func.
2422          */
2423         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2424                                         FM10K_XCAST_MODE_NONE);
2425
2426         fm10k_mbx_unlock(hw);
2427
2428         /* Add default mac address */
2429         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2430                 MAIN_VSI_POOL_NUMBER);
2431
2432         return 0;
2433 }
2434
2435 static int
2436 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2437 {
2438         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2439
2440         PMD_INIT_FUNC_TRACE();
2441
2442         /* only uninitialize in the primary process */
2443         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2444                 return 0;
2445
2446         /* safe to close dev here */
2447         fm10k_dev_close(dev);
2448
2449         dev->dev_ops = NULL;
2450         dev->rx_pkt_burst = NULL;
2451         dev->tx_pkt_burst = NULL;
2452
2453         /* disable uio/vfio intr */
2454         rte_intr_disable(&(dev->pci_dev->intr_handle));
2455
2456         /*PF/VF has different interrupt handling mechanism */
2457         if (hw->mac.type == fm10k_mac_pf) {
2458                 /* disable interrupt */
2459                 fm10k_dev_disable_intr_pf(dev);
2460
2461                 /* unregister callback func to eal lib */
2462                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2463                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2464         } else {
2465                 /* disable interrupt */
2466                 fm10k_dev_disable_intr_vf(dev);
2467
2468                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2469                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2470         }
2471
2472         /* free mac memory */
2473         if (dev->data->mac_addrs) {
2474                 rte_free(dev->data->mac_addrs);
2475                 dev->data->mac_addrs = NULL;
2476         }
2477
2478         memset(hw, 0, sizeof(*hw));
2479
2480         return 0;
2481 }
2482
2483 /*
2484  * The set of PCI devices this driver supports. This driver will enable both PF
2485  * and SRIOV-VF devices.
2486  */
2487 static const struct rte_pci_id pci_id_fm10k_map[] = {
2488 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2489 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2490 #include "rte_pci_dev_ids.h"
2491         { .vendor_id = 0, /* sentinel */ },
2492 };
2493
2494 static struct eth_driver rte_pmd_fm10k = {
2495         .pci_drv = {
2496                 .name = "rte_pmd_fm10k",
2497                 .id_table = pci_id_fm10k_map,
2498                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2499         },
2500         .eth_dev_init = eth_fm10k_dev_init,
2501         .eth_dev_uninit = eth_fm10k_dev_uninit,
2502         .dev_private_size = sizeof(struct fm10k_adapter),
2503 };
2504
2505 /*
2506  * Driver initialization routine.
2507  * Invoked once at EAL init time.
2508  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2509  */
2510 static int
2511 rte_pmd_fm10k_init(__rte_unused const char *name,
2512         __rte_unused const char *params)
2513 {
2514         PMD_INIT_FUNC_TRACE();
2515         rte_eth_driver_register(&rte_pmd_fm10k);
2516         return 0;
2517 }
2518
2519 static struct rte_driver rte_fm10k_driver = {
2520         .type = PMD_PDEV,
2521         .init = rte_pmd_fm10k_init,
2522 };
2523
2524 PMD_REGISTER_DRIVER(rte_fm10k_driver);