drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  64 static int
  65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  67         const u8 *mac, bool add, uint32_t pool);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         q->rxrearm_start = 0;
 125         q->rxrearm_nb = 0;
 126
 127         return 0;
 128 }
 129
 130 /*
 131  * clean queue, descriptor rings, free software buffers used when stopping
 132  * device.
 133  */
 134 static inline void
 135 rx_queue_clean(struct fm10k_rx_queue *q)
 136 {
 137         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 138         uint32_t i;
 139         PMD_INIT_FUNC_TRACE();
 140
 141         /* zero descriptor rings */
 142         for (i = 0; i < q->nb_desc; ++i)
 143                 q->hw_ring[i] = zero;
 144
 145         /* free software buffers */
 146         for (i = 0; i < q->nb_desc; ++i) {
 147                 if (q->sw_ring[i]) {
 148                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 149                         q->sw_ring[i] = NULL;
 150                 }
 151         }
 152 }
 153
 154 /*
 155  * free all queue memory used when releasing the queue (i.e. configure)
 156  */
 157 static inline void
 158 rx_queue_free(struct fm10k_rx_queue *q)
 159 {
 160         PMD_INIT_FUNC_TRACE();
 161         if (q) {
 162                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 163                 rx_queue_clean(q);
 164                 if (q->sw_ring) {
 165                         rte_free(q->sw_ring);
 166                         q->sw_ring = NULL;
 167                 }
 168                 rte_free(q);
 169                 q = NULL;
 170         }
 171 }
 172
 173 /*
 174  * disable RX queue, wait unitl HW finished necessary flush operation
 175  */
 176 static inline int
 177 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 178 {
 179         uint32_t reg, i;
 180
 181         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 182         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 183                         reg & ~FM10K_RXQCTL_ENABLE);
 184
 185         /* Wait 100us at most */
 186         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 187                 rte_delay_us(1);
 188                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 189                 if (!(reg & FM10K_RXQCTL_ENABLE))
 190                         break;
 191         }
 192
 193         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 194                 return -1;
 195
 196         return 0;
 197 }
 198
 199 /*
 200  * reset queue to initial state, allocate software buffers used when starting
 201  * device
 202  */
 203 static inline void
 204 tx_queue_reset(struct fm10k_tx_queue *q)
 205 {
 206         PMD_INIT_FUNC_TRACE();
 207         q->last_free = 0;
 208         q->next_free = 0;
 209         q->nb_used = 0;
 210         q->nb_free = q->nb_desc - 1;
 211         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 212         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 213 }
 214
 215 /*
 216  * clean queue, descriptor rings, free software buffers used when stopping
 217  * device
 218  */
 219 static inline void
 220 tx_queue_clean(struct fm10k_tx_queue *q)
 221 {
 222         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 223         uint32_t i;
 224         PMD_INIT_FUNC_TRACE();
 225
 226         /* zero descriptor rings */
 227         for (i = 0; i < q->nb_desc; ++i)
 228                 q->hw_ring[i] = zero;
 229
 230         /* free software buffers */
 231         for (i = 0; i < q->nb_desc; ++i) {
 232                 if (q->sw_ring[i]) {
 233                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 234                         q->sw_ring[i] = NULL;
 235                 }
 236         }
 237 }
 238
 239 /*
 240  * free all queue memory used when releasing the queue (i.e. configure)
 241  */
 242 static inline void
 243 tx_queue_free(struct fm10k_tx_queue *q)
 244 {
 245         PMD_INIT_FUNC_TRACE();
 246         if (q) {
 247                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 248                 tx_queue_clean(q);
 249                 if (q->rs_tracker.list) {
 250                         rte_free(q->rs_tracker.list);
 251                         q->rs_tracker.list = NULL;
 252                 }
 253                 if (q->sw_ring) {
 254                         rte_free(q->sw_ring);
 255                         q->sw_ring = NULL;
 256                 }
 257                 rte_free(q);
 258                 q = NULL;
 259         }
 260 }
 261
 262 /*
 263  * disable TX queue, wait unitl HW finished necessary flush operation
 264  */
 265 static inline int
 266 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 267 {
 268         uint32_t reg, i;
 269
 270         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 271         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 272                         reg & ~FM10K_TXDCTL_ENABLE);
 273
 274         /* Wait 100us at most */
 275         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 276                 rte_delay_us(1);
 277                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 278                 if (!(reg & FM10K_TXDCTL_ENABLE))
 279                         break;
 280         }
 281
 282         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 283                 return -1;
 284
 285         return 0;
 286 }
 287
 288 static int
 289 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 290 {
 291         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 292         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 293         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 294         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 295
 296         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 297
 298         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 299                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 300                 return -EINVAL;
 301         }
 302
 303         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 304                 return 0;
 305
 306         if (hw->mac.type == fm10k_mac_vf) {
 307                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 308                 return -EINVAL;
 309         }
 310
 311         /* Check VMDQ queue pool number */
 312         if (vmdq_conf->nb_queue_pools >
 313                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 314                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 315                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 316                         vmdq_conf->nb_queue_pools);
 317                 return -EINVAL;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static int
 324 fm10k_dev_configure(struct rte_eth_dev *dev)
 325 {
 326         int ret;
 327
 328         PMD_INIT_FUNC_TRACE();
 329
 330         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 331                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 332         /* multipe queue mode checking */
 333         ret  = fm10k_check_mq_mode(dev);
 334         if (ret != 0) {
 335                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 336                             ret);
 337                 return ret;
 338         }
 339
 340         return 0;
 341 }
 342
 343 /* fls = find last set bit = 32 minus the number of leading zeros */
 344 #ifndef fls
 345 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 346 #endif
 347
 348 static void
 349 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 350 {
 351         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 352         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 353         uint32_t i;
 354
 355         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 356
 357         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 358                 if (!vmdq_conf->pool_map[i].pools)
 359                         continue;
 360                 fm10k_mbx_lock(hw);
 361                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 362                 fm10k_mbx_unlock(hw);
 363         }
 364 }
 365
 366 static void
 367 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 368 {
 369         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 370
 371         /* Add default mac address */
 372         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 373                 MAIN_VSI_POOL_NUMBER);
 374 }
 375
 376 static void
 377 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 378 {
 379         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 380         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 381         uint32_t mrqc, *key, i, reta, j;
 382         uint64_t hf;
 383
 384 #define RSS_KEY_SIZE 40
 385         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 386                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 387                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 388                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 389                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 390                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 391         };
 392
 393         if (dev->data->nb_rx_queues == 1 ||
 394             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 395             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 396                 return;
 397
 398         /* random key is rss_intel_key (default) or user provided (rss_key) */
 399         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 400                 key = (uint32_t *)rss_intel_key;
 401         else
 402                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 403
 404         /* Now fill our hash function seeds, 4 bytes at a time */
 405         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 406                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 407
 408         /*
 409          * Fill in redirection table
 410          * The byte-swap is needed because NIC registers are in
 411          * little-endian order.
 412          */
 413         reta = 0;
 414         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 415                 if (j == dev->data->nb_rx_queues)
 416                         j = 0;
 417                 reta = (reta << CHAR_BIT) | j;
 418                 if ((i & 3) == 3)
 419                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 420                                         rte_bswap32(reta));
 421         }
 422
 423         /*
 424          * Generate RSS hash based on packet types, TCP/UDP
 425          * port numbers and/or IPv4/v6 src and dst addresses
 426          */
 427         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 428         mrqc = 0;
 429         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 430         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 431         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 432         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 433         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 434         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 435         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 436         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 437         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 438
 439         if (mrqc == 0) {
 440                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 441                         "supported", hf);
 442                 return;
 443         }
 444
 445         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 446 }
 447
 448 static void
 449 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 450         uint16_t nb_lport_old, uint16_t nb_lport_new)
 451 {
 452         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 453         uint32_t i;
 454
 455         fm10k_mbx_lock(hw);
 456         /* Disable previous logic ports */
 457         if (nb_lport_old)
 458                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 459                         nb_lport_old, false);
 460         /* Enable new logic ports */
 461         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 462                 nb_lport_new, true);
 463         fm10k_mbx_unlock(hw);
 464
 465         for (i = 0; i < nb_lport_new; i++) {
 466                 /* Set unicast mode by default. App can change
 467                  * to other mode in other API func.
 468                  */
 469                 fm10k_mbx_lock(hw);
 470                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 471                         FM10K_XCAST_MODE_NONE);
 472                 fm10k_mbx_unlock(hw);
 473         }
 474 }
 475
 476 static void
 477 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 478 {
 479         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 480         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 481         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 482         struct fm10k_macvlan_filter_info *macvlan;
 483         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 484         uint16_t nb_lport_new, nb_lport_old;
 485
 486         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 487         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 488
 489         fm10k_dev_rss_configure(dev);
 490
 491         /* only PF supports VMDQ */
 492         if (hw->mac.type != fm10k_mac_pf)
 493                 return;
 494
 495         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 496                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 497
 498         /* no pool number change, no need to update logic port and VLAN/MAC */
 499         if (macvlan->nb_queue_pools == nb_queue_pools)
 500                 return;
 501
 502         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 503         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 504         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 505
 506         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 507         memset(dev->data->mac_addrs, 0,
 508                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 509         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 510                 &dev->data->mac_addrs[0]);
 511         memset(macvlan, 0, sizeof(*macvlan));
 512         macvlan->nb_queue_pools = nb_queue_pools;
 513
 514         if (nb_queue_pools)
 515                 fm10k_dev_vmdq_rx_configure(dev);
 516         else
 517                 fm10k_dev_pf_main_vsi_reset(dev);
 518 }
 519
 520 static int
 521 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 522 {
 523         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 524         int i, ret;
 525         struct fm10k_tx_queue *txq;
 526         uint64_t base_addr;
 527         uint32_t size;
 528
 529         /* Disable TXINT to avoid possible interrupt */
 530         for (i = 0; i < hw->mac.max_queues; i++)
 531                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 532                                 3 << FM10K_TXINT_TIMER_SHIFT);
 533
 534         /* Setup TX queue */
 535         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 536                 txq = dev->data->tx_queues[i];
 537                 base_addr = txq->hw_ring_phys_addr;
 538                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 539
 540                 /* disable queue to avoid issues while updating state */
 541                 ret = tx_queue_disable(hw, i);
 542                 if (ret) {
 543                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 544                         return -1;
 545                 }
 546
 547                 /* set location and size for descriptor ring */
 548                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 549                                 base_addr & UINT64_LOWER_32BITS_MASK);
 550                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 551                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 552                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 553         }
 554         return 0;
 555 }
 556
 557 static int
 558 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 559 {
 560         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 561         int i, ret;
 562         struct fm10k_rx_queue *rxq;
 563         uint64_t base_addr;
 564         uint32_t size;
 565         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 566         uint16_t buf_size;
 567
 568         /* Disable RXINT to avoid possible interrupt */
 569         for (i = 0; i < hw->mac.max_queues; i++)
 570                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 571                                 3 << FM10K_RXINT_TIMER_SHIFT);
 572
 573         /* Setup RX queues */
 574         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 575                 rxq = dev->data->rx_queues[i];
 576                 base_addr = rxq->hw_ring_phys_addr;
 577                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 578
 579                 /* disable queue to avoid issues while updating state */
 580                 ret = rx_queue_disable(hw, i);
 581                 if (ret) {
 582                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 583                         return -1;
 584                 }
 585
 586                 /* Setup the Base and Length of the Rx Descriptor Ring */
 587                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 588                                 base_addr & UINT64_LOWER_32BITS_MASK);
 589                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 590                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 591                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 592
 593                 /* Configure the Rx buffer size for one buff without split */
 594                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 595                         RTE_PKTMBUF_HEADROOM);
 596                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 597                  * reserved for this purpose, and the worst case could be 511B.
 598                  * But SRR reg assumes all buffers have the same size. In order
 599                  * to fill the gap, we'll have to consider the worst case and
 600                  * assume 512B is reserved. If we don't do so, it's possible
 601                  * for HW to overwrite data to next mbuf.
 602                  */
 603                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 604
 605                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 606                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 607
 608                 /* It adds dual VLAN length for supporting dual VLAN */
 609                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 610                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 611                         dev->data->dev_conf.rxmode.enable_scatter) {
 612                         uint32_t reg;
 613                         dev->data->scattered_rx = 1;
 614                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 615                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 616                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 617                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 618                 }
 619
 620                 /* Enable drop on empty, it's RO for VF */
 621                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 622                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 623
 624                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 625                 FM10K_WRITE_FLUSH(hw);
 626         }
 627
 628         /* Configure VMDQ/RSS if applicable */
 629         fm10k_dev_mq_rx_configure(dev);
 630         return 0;
 631 }
 632
 633 static int
 634 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 635 {
 636         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 637         int err = -1;
 638         uint32_t reg;
 639         struct fm10k_rx_queue *rxq;
 640
 641         PMD_INIT_FUNC_TRACE();
 642
 643         if (rx_queue_id < dev->data->nb_rx_queues) {
 644                 rxq = dev->data->rx_queues[rx_queue_id];
 645                 err = rx_queue_reset(rxq);
 646                 if (err == -ENOMEM) {
 647                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 648                         return err;
 649                 } else if (err == -EINVAL) {
 650                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 651                                 " %d", err);
 652                         return err;
 653                 }
 654
 655                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 656                  * Note: this must be done AFTER the queue is enabled on real
 657                  * hardware, but BEFORE the queue is enabled when using the
 658                  * emulation platform. Do it in both places for now and remove
 659                  * this comment and the following two register writes when the
 660                  * emulation platform is no longer being used.
 661                  */
 662                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 663                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 664
 665                 /* Set PF ownership flag for PF devices */
 666                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 667                 if (hw->mac.type == fm10k_mac_pf)
 668                         reg |= FM10K_RXQCTL_PF;
 669                 reg |= FM10K_RXQCTL_ENABLE;
 670                 /* enable RX queue */
 671                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 672                 FM10K_WRITE_FLUSH(hw);
 673
 674                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 675                  * Note: this must be done AFTER the queue is enabled
 676                  */
 677                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 678                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 679         }
 680
 681         return err;
 682 }
 683
 684 static int
 685 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 686 {
 687         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 688
 689         PMD_INIT_FUNC_TRACE();
 690
 691         if (rx_queue_id < dev->data->nb_rx_queues) {
 692                 /* Disable RX queue */
 693                 rx_queue_disable(hw, rx_queue_id);
 694
 695                 /* Free mbuf and clean HW ring */
 696                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 697         }
 698
 699         return 0;
 700 }
 701
 702 static int
 703 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 704 {
 705         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 706         /** @todo - this should be defined in the shared code */
 707 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 708         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 709         int err = 0;
 710
 711         PMD_INIT_FUNC_TRACE();
 712
 713         if (tx_queue_id < dev->data->nb_tx_queues) {
 714                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 715
 716                 /* reset head and tail pointers */
 717                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 718                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 719
 720                 /* enable TX queue */
 721                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 722                                         FM10K_TXDCTL_ENABLE | txdctl);
 723                 FM10K_WRITE_FLUSH(hw);
 724         } else
 725                 err = -1;
 726
 727         return err;
 728 }
 729
 730 static int
 731 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 732 {
 733         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 734
 735         PMD_INIT_FUNC_TRACE();
 736
 737         if (tx_queue_id < dev->data->nb_tx_queues) {
 738                 tx_queue_disable(hw, tx_queue_id);
 739                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 740         }
 741
 742         return 0;
 743 }
 744
 745 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 746 {
 747         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 748                 != FM10K_DGLORTMAP_NONE);
 749 }
 750
 751 static void
 752 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 753 {
 754         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 755         int status;
 756
 757         PMD_INIT_FUNC_TRACE();
 758
 759         /* Return if it didn't acquire valid glort range */
 760         if (!fm10k_glort_valid(hw))
 761                 return;
 762
 763         fm10k_mbx_lock(hw);
 764         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 765                                 FM10K_XCAST_MODE_PROMISC);
 766         fm10k_mbx_unlock(hw);
 767
 768         if (status != FM10K_SUCCESS)
 769                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 770 }
 771
 772 static void
 773 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 774 {
 775         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 776         uint8_t mode;
 777         int status;
 778
 779         PMD_INIT_FUNC_TRACE();
 780
 781         /* Return if it didn't acquire valid glort range */
 782         if (!fm10k_glort_valid(hw))
 783                 return;
 784
 785         if (dev->data->all_multicast == 1)
 786                 mode = FM10K_XCAST_MODE_ALLMULTI;
 787         else
 788                 mode = FM10K_XCAST_MODE_NONE;
 789
 790         fm10k_mbx_lock(hw);
 791         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 792                                 mode);
 793         fm10k_mbx_unlock(hw);
 794
 795         if (status != FM10K_SUCCESS)
 796                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 797 }
 798
 799 static void
 800 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 801 {
 802         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 803         int status;
 804
 805         PMD_INIT_FUNC_TRACE();
 806
 807         /* Return if it didn't acquire valid glort range */
 808         if (!fm10k_glort_valid(hw))
 809                 return;
 810
 811         /* If promiscuous mode is enabled, it doesn't make sense to enable
 812          * allmulticast and disable promiscuous since fm10k only can select
 813          * one of the modes.
 814          */
 815         if (dev->data->promiscuous) {
 816                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 817                         "needn't enable allmulticast");
 818                 return;
 819         }
 820
 821         fm10k_mbx_lock(hw);
 822         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 823                                 FM10K_XCAST_MODE_ALLMULTI);
 824         fm10k_mbx_unlock(hw);
 825
 826         if (status != FM10K_SUCCESS)
 827                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 828 }
 829
 830 static void
 831 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 832 {
 833         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 834         int status;
 835
 836         PMD_INIT_FUNC_TRACE();
 837
 838         /* Return if it didn't acquire valid glort range */
 839         if (!fm10k_glort_valid(hw))
 840                 return;
 841
 842         if (dev->data->promiscuous) {
 843                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 844                         "since promisc mode is enabled");
 845                 return;
 846         }
 847
 848         fm10k_mbx_lock(hw);
 849         /* Change mode to unicast mode */
 850         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 851                                 FM10K_XCAST_MODE_NONE);
 852         fm10k_mbx_unlock(hw);
 853
 854         if (status != FM10K_SUCCESS)
 855                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 856 }
 857
 858 static void
 859 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 860 {
 861         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 862         uint32_t dglortdec, pool_len, rss_len, i;
 863         uint16_t nb_queue_pools;
 864         struct fm10k_macvlan_filter_info *macvlan;
 865
 866         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 867         nb_queue_pools = macvlan->nb_queue_pools;
 868         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 869         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 870         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 871
 872         /* Establish only MAP 0 as valid */
 873         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 874
 875         /* Configure VMDQ/RSS DGlort Decoder */
 876         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 877
 878         /* Invalidate all other GLORT entries */
 879         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 880                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 881                                 FM10K_DGLORTMAP_NONE);
 882 }
 883
 884 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 885 static int
 886 fm10k_dev_start(struct rte_eth_dev *dev)
 887 {
 888         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 889         int i, diag;
 890
 891         PMD_INIT_FUNC_TRACE();
 892
 893         /* stop, init, then start the hw */
 894         diag = fm10k_stop_hw(hw);
 895         if (diag != FM10K_SUCCESS) {
 896                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 897                 return -EIO;
 898         }
 899
 900         diag = fm10k_init_hw(hw);
 901         if (diag != FM10K_SUCCESS) {
 902                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 903                 return -EIO;
 904         }
 905
 906         diag = fm10k_start_hw(hw);
 907         if (diag != FM10K_SUCCESS) {
 908                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 909                 return -EIO;
 910         }
 911
 912         diag = fm10k_dev_tx_init(dev);
 913         if (diag) {
 914                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 915                 return diag;
 916         }
 917
 918         diag = fm10k_dev_rx_init(dev);
 919         if (diag) {
 920                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 921                 return diag;
 922         }
 923
 924         if (hw->mac.type == fm10k_mac_pf)
 925                 fm10k_dev_dglort_map_configure(dev);
 926
 927         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 928                 struct fm10k_rx_queue *rxq;
 929                 rxq = dev->data->rx_queues[i];
 930
 931                 if (rxq->rx_deferred_start)
 932                         continue;
 933                 diag = fm10k_dev_rx_queue_start(dev, i);
 934                 if (diag != 0) {
 935                         int j;
 936                         for (j = 0; j < i; ++j)
 937                                 rx_queue_clean(dev->data->rx_queues[j]);
 938                         return diag;
 939                 }
 940         }
 941
 942         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 943                 struct fm10k_tx_queue *txq;
 944                 txq = dev->data->tx_queues[i];
 945
 946                 if (txq->tx_deferred_start)
 947                         continue;
 948                 diag = fm10k_dev_tx_queue_start(dev, i);
 949                 if (diag != 0) {
 950                         int j;
 951                         for (j = 0; j < i; ++j)
 952                                 tx_queue_clean(dev->data->tx_queues[j]);
 953                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 954                                 rx_queue_clean(dev->data->rx_queues[j]);
 955                         return diag;
 956                 }
 957         }
 958
 959         /* Update default vlan when not in VMDQ mode */
 960         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 961                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 962
 963         return 0;
 964 }
 965
 966 static void
 967 fm10k_dev_stop(struct rte_eth_dev *dev)
 968 {
 969         int i;
 970
 971         PMD_INIT_FUNC_TRACE();
 972
 973         if (dev->data->tx_queues)
 974                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 975                         fm10k_dev_tx_queue_stop(dev, i);
 976
 977         if (dev->data->rx_queues)
 978                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 979                         fm10k_dev_rx_queue_stop(dev, i);
 980 }
 981
 982 static void
 983 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 984 {
 985         int i;
 986
 987         PMD_INIT_FUNC_TRACE();
 988
 989         if (dev->data->tx_queues) {
 990                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 991                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 992         }
 993
 994         if (dev->data->rx_queues) {
 995                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 996                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 997         }
 998 }
 999
1000 static void
1001 fm10k_dev_close(struct rte_eth_dev *dev)
1002 {
1003         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1004         uint16_t nb_lport;
1005         struct fm10k_macvlan_filter_info *macvlan;
1006
1007         PMD_INIT_FUNC_TRACE();
1008
1009         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1010         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1011         fm10k_mbx_lock(hw);
1012         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1013                 nb_lport, false);
1014         fm10k_mbx_unlock(hw);
1015
1016         /* Stop mailbox service first */
1017         fm10k_close_mbx_service(hw);
1018         fm10k_dev_stop(dev);
1019         fm10k_dev_queue_release(dev);
1020         fm10k_stop_hw(hw);
1021 }
1022
1023 static int
1024 fm10k_link_update(struct rte_eth_dev *dev,
1025         __rte_unused int wait_to_complete)
1026 {
1027         PMD_INIT_FUNC_TRACE();
1028
1029         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1030          * x8 PCIe interface. For now, we leave the speed undefined since there
1031          * is no 50Gbps Ethernet. */
1032         dev->data->dev_link.link_speed  = 0;
1033         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1034         dev->data->dev_link.link_status = 1;
1035
1036         return 0;
1037 }
1038
1039 static void
1040 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1041 {
1042         uint64_t ipackets, opackets, ibytes, obytes;
1043         struct fm10k_hw *hw =
1044                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1045         struct fm10k_hw_stats *hw_stats =
1046                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1047         int i;
1048
1049         PMD_INIT_FUNC_TRACE();
1050
1051         fm10k_update_hw_stats(hw, hw_stats);
1052
1053         ipackets = opackets = ibytes = obytes = 0;
1054         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1055                 (i < hw->mac.max_queues); ++i) {
1056                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1057                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1058                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1059                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1060                 ipackets += stats->q_ipackets[i];
1061                 opackets += stats->q_opackets[i];
1062                 ibytes   += stats->q_ibytes[i];
1063                 obytes   += stats->q_obytes[i];
1064         }
1065         stats->ipackets = ipackets;
1066         stats->opackets = opackets;
1067         stats->ibytes = ibytes;
1068         stats->obytes = obytes;
1069 }
1070
1071 static void
1072 fm10k_stats_reset(struct rte_eth_dev *dev)
1073 {
1074         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1075         struct fm10k_hw_stats *hw_stats =
1076                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1077
1078         PMD_INIT_FUNC_TRACE();
1079
1080         memset(hw_stats, 0, sizeof(*hw_stats));
1081         fm10k_rebind_hw_stats(hw, hw_stats);
1082 }
1083
1084 static void
1085 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1086         struct rte_eth_dev_info *dev_info)
1087 {
1088         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1089
1090         PMD_INIT_FUNC_TRACE();
1091
1092         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1093         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1094         dev_info->max_rx_queues      = hw->mac.max_queues;
1095         dev_info->max_tx_queues      = hw->mac.max_queues;
1096         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1097         dev_info->max_hash_mac_addrs = 0;
1098         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1099         dev_info->vmdq_pool_base     = 0;
1100         dev_info->vmdq_queue_base    = 0;
1101         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1102         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1103         dev_info->rx_offload_capa =
1104                 DEV_RX_OFFLOAD_VLAN_STRIP |
1105                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1106                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1107                 DEV_RX_OFFLOAD_TCP_CKSUM;
1108         dev_info->tx_offload_capa =
1109                 DEV_TX_OFFLOAD_VLAN_INSERT |
1110                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1111                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1112                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1113                 DEV_TX_OFFLOAD_TCP_TSO;
1114
1115         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1116         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1117
1118         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1119                 .rx_thresh = {
1120                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1121                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1122                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1123                 },
1124                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1125                 .rx_drop_en = 0,
1126         };
1127
1128         dev_info->default_txconf = (struct rte_eth_txconf) {
1129                 .tx_thresh = {
1130                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1131                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1132                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1133                 },
1134                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1135                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1136                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1137                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1138         };
1139
1140         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1141                 .nb_max = FM10K_MAX_RX_DESC,
1142                 .nb_min = FM10K_MIN_RX_DESC,
1143                 .nb_align = FM10K_MULT_RX_DESC,
1144         };
1145
1146         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1147                 .nb_max = FM10K_MAX_TX_DESC,
1148                 .nb_min = FM10K_MIN_TX_DESC,
1149                 .nb_align = FM10K_MULT_TX_DESC,
1150         };
1151 }
1152
1153 static int
1154 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1155 {
1156         s32 result;
1157         uint16_t mac_num = 0;
1158         uint32_t vid_idx, vid_bit, mac_index;
1159         struct fm10k_hw *hw;
1160         struct fm10k_macvlan_filter_info *macvlan;
1161         struct rte_eth_dev_data *data = dev->data;
1162
1163         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1164         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1165
1166         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1167                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1168                 return (-EINVAL);
1169         }
1170
1171         if (vlan_id > ETH_VLAN_ID_MAX) {
1172                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1173                 return (-EINVAL);
1174         }
1175
1176         vid_idx = FM10K_VFTA_IDX(vlan_id);
1177         vid_bit = FM10K_VFTA_BIT(vlan_id);
1178         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1179         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1180                 return 0;
1181         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1182         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1183                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1184                         "in the VLAN filter table");
1185                 return (-EINVAL);
1186         }
1187
1188         fm10k_mbx_lock(hw);
1189         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1190         fm10k_mbx_unlock(hw);
1191         if (result != FM10K_SUCCESS) {
1192                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1193                 return (-EIO);
1194         }
1195
1196         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1197                         (result == FM10K_SUCCESS); mac_index++) {
1198                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1199                         continue;
1200                 if (mac_num > macvlan->mac_num - 1) {
1201                         PMD_INIT_LOG(ERR, "MAC address number "
1202                                         "not match");
1203                         break;
1204                 }
1205                 fm10k_mbx_lock(hw);
1206                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1207                         data->mac_addrs[mac_index].addr_bytes,
1208                         vlan_id, on, 0);
1209                 fm10k_mbx_unlock(hw);
1210                 mac_num++;
1211         }
1212         if (result != FM10K_SUCCESS) {
1213                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1214                 return (-EIO);
1215         }
1216
1217         if (on) {
1218                 macvlan->vlan_num++;
1219                 macvlan->vfta[vid_idx] |= vid_bit;
1220         } else {
1221                 macvlan->vlan_num--;
1222                 macvlan->vfta[vid_idx] &= ~vid_bit;
1223         }
1224         return 0;
1225 }
1226
1227 static void
1228 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1229 {
1230         if (mask & ETH_VLAN_STRIP_MASK) {
1231                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1232                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1233                                         "always on in fm10k");
1234         }
1235
1236         if (mask & ETH_VLAN_EXTEND_MASK) {
1237                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1238                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1239                                         "supported in fm10k");
1240         }
1241
1242         if (mask & ETH_VLAN_FILTER_MASK) {
1243                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1244                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1245         }
1246 }
1247
1248 /* Add/Remove a MAC address, and update filters to main VSI */
1249 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1250                 const u8 *mac, bool add, uint32_t pool)
1251 {
1252         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1253         struct fm10k_macvlan_filter_info *macvlan;
1254         uint32_t i, j, k;
1255
1256         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1257
1258         if (pool != MAIN_VSI_POOL_NUMBER) {
1259                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1260                         "mac to pool %u", pool);
1261                 return;
1262         }
1263         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1264                 if (!macvlan->vfta[j])
1265                         continue;
1266                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1267                         if (!(macvlan->vfta[j] & (1 << k)))
1268                                 continue;
1269                         if (i + 1 > macvlan->vlan_num) {
1270                                 PMD_INIT_LOG(ERR, "vlan number not match");
1271                                 return;
1272                         }
1273                         fm10k_mbx_lock(hw);
1274                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1275                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1276                         fm10k_mbx_unlock(hw);
1277                         i++;
1278                 }
1279         }
1280 }
1281
1282 /* Add/Remove a MAC address, and update filters to VMDQ */
1283 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1284                 const u8 *mac, bool add, uint32_t pool)
1285 {
1286         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1287         struct fm10k_macvlan_filter_info *macvlan;
1288         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1289         uint32_t i;
1290
1291         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1292         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1293
1294         if (pool > macvlan->nb_queue_pools) {
1295                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1296                         " Max pool is %u",
1297                         pool, macvlan->nb_queue_pools);
1298                 return;
1299         }
1300         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1301                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1302                         continue;
1303                 fm10k_mbx_lock(hw);
1304                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1305                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1306                 fm10k_mbx_unlock(hw);
1307         }
1308 }
1309
1310 /* Add/Remove a MAC address, and update filters */
1311 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1312                 const u8 *mac, bool add, uint32_t pool)
1313 {
1314         struct fm10k_macvlan_filter_info *macvlan;
1315
1316         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1317
1318         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1319                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1320         else
1321                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1322
1323         if (add)
1324                 macvlan->mac_num++;
1325         else
1326                 macvlan->mac_num--;
1327 }
1328
1329 /* Add a MAC address, and update filters */
1330 static void
1331 fm10k_macaddr_add(struct rte_eth_dev *dev,
1332                 struct ether_addr *mac_addr,
1333                 uint32_t index,
1334                 uint32_t pool)
1335 {
1336         struct fm10k_macvlan_filter_info *macvlan;
1337
1338         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1339         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1340         macvlan->mac_vmdq_id[index] = pool;
1341 }
1342
1343 /* Remove a MAC address, and update filters */
1344 static void
1345 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1346 {
1347         struct rte_eth_dev_data *data = dev->data;
1348         struct fm10k_macvlan_filter_info *macvlan;
1349
1350         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1351         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1352                         FALSE, macvlan->mac_vmdq_id[index]);
1353         macvlan->mac_vmdq_id[index] = 0;
1354 }
1355
1356 static inline int
1357 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1358 {
1359         if ((request < min) || (request > max) || ((request % mult) != 0))
1360                 return -1;
1361         else
1362                 return 0;
1363 }
1364
1365 /*
1366  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1367  * the physical address is required. If the memzone is already created, then
1368  * this function returns a pointer to the existing memzone.
1369  */
1370 static inline const struct rte_memzone *
1371 allocate_hw_ring(const char *driver_name, const char *ring_name,
1372         uint8_t port_id, uint16_t queue_id, int socket_id,
1373         uint32_t size, uint32_t align)
1374 {
1375         char name[RTE_MEMZONE_NAMESIZE];
1376         const struct rte_memzone *mz;
1377
1378         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1379                  driver_name, ring_name, port_id, queue_id, socket_id);
1380
1381         /* return the memzone if it already exists */
1382         mz = rte_memzone_lookup(name);
1383         if (mz)
1384                 return mz;
1385
1386 #ifdef RTE_LIBRTE_XEN_DOM0
1387         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1388                                            RTE_PGSIZE_2M);
1389 #else
1390         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1391 #endif
1392 }
1393
1394 static inline int
1395 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1396 {
1397         if ((request < min) || (request > max) || ((div % request) != 0))
1398                 return -1;
1399         else
1400                 return 0;
1401 }
1402
1403 static inline int
1404 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1405 {
1406         uint16_t rx_free_thresh;
1407
1408         if (conf->rx_free_thresh == 0)
1409                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1410         else
1411                 rx_free_thresh = conf->rx_free_thresh;
1412
1413         /* make sure the requested threshold satisfies the constraints */
1414         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1415                         FM10K_RX_FREE_THRESH_MAX(q),
1416                         FM10K_RX_FREE_THRESH_DIV(q),
1417                         rx_free_thresh)) {
1418                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1419                         "less than or equal to %u, "
1420                         "greater than or equal to %u, "
1421                         "and a divisor of %u",
1422                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1423                         FM10K_RX_FREE_THRESH_MIN(q),
1424                         FM10K_RX_FREE_THRESH_DIV(q));
1425                 return (-EINVAL);
1426         }
1427
1428         q->alloc_thresh = rx_free_thresh;
1429         q->drop_en = conf->rx_drop_en;
1430         q->rx_deferred_start = conf->rx_deferred_start;
1431
1432         return 0;
1433 }
1434
1435 /*
1436  * Hardware requires specific alignment for Rx packet buffers. At
1437  * least one of the following two conditions must be satisfied.
1438  *  1. Address is 512B aligned
1439  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1440  *
1441  * As such, the driver may need to adjust the DMA address within the
1442  * buffer by up to 512B.
1443  *
1444  * return 1 if the element size is valid, otherwise return 0.
1445  */
1446 static int
1447 mempool_element_size_valid(struct rte_mempool *mp)
1448 {
1449         uint32_t min_size;
1450
1451         /* elt_size includes mbuf header and headroom */
1452         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1453                         RTE_PKTMBUF_HEADROOM;
1454
1455         /* account for up to 512B of alignment */
1456         min_size -= FM10K_RX_DATABUF_ALIGN;
1457
1458         /* sanity check for overflow */
1459         if (min_size > mp->elt_size)
1460                 return 0;
1461
1462         /* size is valid */
1463         return 1;
1464 }
1465
1466 static int
1467 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1468         uint16_t nb_desc, unsigned int socket_id,
1469         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1470 {
1471         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1472         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1473         struct fm10k_rx_queue *q;
1474         const struct rte_memzone *mz;
1475
1476         PMD_INIT_FUNC_TRACE();
1477
1478         /* make sure the mempool element size can account for alignment. */
1479         if (!mempool_element_size_valid(mp)) {
1480                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1481                 return (-EINVAL);
1482         }
1483
1484         /* make sure a valid number of descriptors have been requested */
1485         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1486                                 FM10K_MULT_RX_DESC, nb_desc)) {
1487                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1488                         "less than or equal to %"PRIu32", "
1489                         "greater than or equal to %u, "
1490                         "and a multiple of %u",
1491                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1492                         FM10K_MULT_RX_DESC);
1493                 return (-EINVAL);
1494         }
1495
1496         /*
1497          * if this queue existed already, free the associated memory. The
1498          * queue cannot be reused in case we need to allocate memory on
1499          * different socket than was previously used.
1500          */
1501         if (dev->data->rx_queues[queue_id] != NULL) {
1502                 rx_queue_free(dev->data->rx_queues[queue_id]);
1503                 dev->data->rx_queues[queue_id] = NULL;
1504         }
1505
1506         /* allocate memory for the queue structure */
1507         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1508                                 socket_id);
1509         if (q == NULL) {
1510                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1511                 return (-ENOMEM);
1512         }
1513
1514         /* setup queue */
1515         q->mp = mp;
1516         q->nb_desc = nb_desc;
1517         q->port_id = dev->data->port_id;
1518         q->queue_id = queue_id;
1519         q->tail_ptr = (volatile uint32_t *)
1520                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1521         if (handle_rxconf(q, conf))
1522                 return (-EINVAL);
1523
1524         /* allocate memory for the software ring */
1525         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1526                                         nb_desc * sizeof(struct rte_mbuf *),
1527                                         RTE_CACHE_LINE_SIZE, socket_id);
1528         if (q->sw_ring == NULL) {
1529                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1530                 rte_free(q);
1531                 return (-ENOMEM);
1532         }
1533
1534         /*
1535          * allocate memory for the hardware descriptor ring. A memzone large
1536          * enough to hold the maximum ring size is requested to allow for
1537          * resizing in later calls to the queue setup function.
1538          */
1539         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1540                                 dev->data->port_id, queue_id, socket_id,
1541                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1542         if (mz == NULL) {
1543                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1544                 rte_free(q->sw_ring);
1545                 rte_free(q);
1546                 return (-ENOMEM);
1547         }
1548         q->hw_ring = mz->addr;
1549 #ifdef RTE_LIBRTE_XEN_DOM0
1550         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1551 #else
1552         q->hw_ring_phys_addr = mz->phys_addr;
1553 #endif
1554
1555         /* Check if number of descs satisfied Vector requirement */
1556         if (!rte_is_power_of_2(nb_desc)) {
1557                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1558                                     "preconditions - canceling the feature for "
1559                                     "the whole port[%d]",
1560                              q->queue_id, q->port_id);
1561                 dev_info->rx_vec_allowed = false;
1562         } else
1563                 fm10k_rxq_vec_setup(q);
1564
1565         dev->data->rx_queues[queue_id] = q;
1566         return 0;
1567 }
1568
1569 static void
1570 fm10k_rx_queue_release(void *queue)
1571 {
1572         PMD_INIT_FUNC_TRACE();
1573
1574         rx_queue_free(queue);
1575 }
1576
1577 static inline int
1578 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1579 {
1580         uint16_t tx_free_thresh;
1581         uint16_t tx_rs_thresh;
1582
1583         /* constraint MACROs require that tx_free_thresh is configured
1584          * before tx_rs_thresh */
1585         if (conf->tx_free_thresh == 0)
1586                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1587         else
1588                 tx_free_thresh = conf->tx_free_thresh;
1589
1590         /* make sure the requested threshold satisfies the constraints */
1591         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1592                         FM10K_TX_FREE_THRESH_MAX(q),
1593                         FM10K_TX_FREE_THRESH_DIV(q),
1594                         tx_free_thresh)) {
1595                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1596                         "less than or equal to %u, "
1597                         "greater than or equal to %u, "
1598                         "and a divisor of %u",
1599                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1600                         FM10K_TX_FREE_THRESH_MIN(q),
1601                         FM10K_TX_FREE_THRESH_DIV(q));
1602                 return (-EINVAL);
1603         }
1604
1605         q->free_thresh = tx_free_thresh;
1606
1607         if (conf->tx_rs_thresh == 0)
1608                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1609         else
1610                 tx_rs_thresh = conf->tx_rs_thresh;
1611
1612         q->tx_deferred_start = conf->tx_deferred_start;
1613
1614         /* make sure the requested threshold satisfies the constraints */
1615         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1616                         FM10K_TX_RS_THRESH_MAX(q),
1617                         FM10K_TX_RS_THRESH_DIV(q),
1618                         tx_rs_thresh)) {
1619                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1620                         "less than or equal to %u, "
1621                         "greater than or equal to %u, "
1622                         "and a divisor of %u",
1623                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1624                         FM10K_TX_RS_THRESH_MIN(q),
1625                         FM10K_TX_RS_THRESH_DIV(q));
1626                 return (-EINVAL);
1627         }
1628
1629         q->rs_thresh = tx_rs_thresh;
1630
1631         return 0;
1632 }
1633
1634 static int
1635 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1636         uint16_t nb_desc, unsigned int socket_id,
1637         const struct rte_eth_txconf *conf)
1638 {
1639         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1640         struct fm10k_tx_queue *q;
1641         const struct rte_memzone *mz;
1642
1643         PMD_INIT_FUNC_TRACE();
1644
1645         /* make sure a valid number of descriptors have been requested */
1646         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1647                                 FM10K_MULT_TX_DESC, nb_desc)) {
1648                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1649                         "less than or equal to %"PRIu32", "
1650                         "greater than or equal to %u, "
1651                         "and a multiple of %u",
1652                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1653                         FM10K_MULT_TX_DESC);
1654                 return (-EINVAL);
1655         }
1656
1657         /*
1658          * if this queue existed already, free the associated memory. The
1659          * queue cannot be reused in case we need to allocate memory on
1660          * different socket than was previously used.
1661          */
1662         if (dev->data->tx_queues[queue_id] != NULL) {
1663                 tx_queue_free(dev->data->tx_queues[queue_id]);
1664                 dev->data->tx_queues[queue_id] = NULL;
1665         }
1666
1667         /* allocate memory for the queue structure */
1668         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1669                                 socket_id);
1670         if (q == NULL) {
1671                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1672                 return (-ENOMEM);
1673         }
1674
1675         /* setup queue */
1676         q->nb_desc = nb_desc;
1677         q->port_id = dev->data->port_id;
1678         q->queue_id = queue_id;
1679         q->tail_ptr = (volatile uint32_t *)
1680                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1681         if (handle_txconf(q, conf))
1682                 return (-EINVAL);
1683
1684         /* allocate memory for the software ring */
1685         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1686                                         nb_desc * sizeof(struct rte_mbuf *),
1687                                         RTE_CACHE_LINE_SIZE, socket_id);
1688         if (q->sw_ring == NULL) {
1689                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1690                 rte_free(q);
1691                 return (-ENOMEM);
1692         }
1693
1694         /*
1695          * allocate memory for the hardware descriptor ring. A memzone large
1696          * enough to hold the maximum ring size is requested to allow for
1697          * resizing in later calls to the queue setup function.
1698          */
1699         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1700                                 dev->data->port_id, queue_id, socket_id,
1701                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1702         if (mz == NULL) {
1703                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1704                 rte_free(q->sw_ring);
1705                 rte_free(q);
1706                 return (-ENOMEM);
1707         }
1708         q->hw_ring = mz->addr;
1709 #ifdef RTE_LIBRTE_XEN_DOM0
1710         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1711 #else
1712         q->hw_ring_phys_addr = mz->phys_addr;
1713 #endif
1714
1715         /*
1716          * allocate memory for the RS bit tracker. Enough slots to hold the
1717          * descriptor index for each RS bit needing to be set are required.
1718          */
1719         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1720                                 ((nb_desc + 1) / q->rs_thresh) *
1721                                 sizeof(uint16_t),
1722                                 RTE_CACHE_LINE_SIZE, socket_id);
1723         if (q->rs_tracker.list == NULL) {
1724                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1725                 rte_free(q->sw_ring);
1726                 rte_free(q);
1727                 return (-ENOMEM);
1728         }
1729
1730         dev->data->tx_queues[queue_id] = q;
1731         return 0;
1732 }
1733
1734 static void
1735 fm10k_tx_queue_release(void *queue)
1736 {
1737         PMD_INIT_FUNC_TRACE();
1738
1739         tx_queue_free(queue);
1740 }
1741
1742 static int
1743 fm10k_reta_update(struct rte_eth_dev *dev,
1744                         struct rte_eth_rss_reta_entry64 *reta_conf,
1745                         uint16_t reta_size)
1746 {
1747         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1748         uint16_t i, j, idx, shift;
1749         uint8_t mask;
1750         uint32_t reta;
1751
1752         PMD_INIT_FUNC_TRACE();
1753
1754         if (reta_size > FM10K_MAX_RSS_INDICES) {
1755                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1756                         "(%d) doesn't match the number hardware can supported "
1757                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1758                 return -EINVAL;
1759         }
1760
1761         /*
1762          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1763          * 128-entries in 32 registers
1764          */
1765         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1766                 idx = i / RTE_RETA_GROUP_SIZE;
1767                 shift = i % RTE_RETA_GROUP_SIZE;
1768                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1769                                 BIT_MASK_PER_UINT32);
1770                 if (mask == 0)
1771                         continue;
1772
1773                 reta = 0;
1774                 if (mask != BIT_MASK_PER_UINT32)
1775                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1776
1777                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1778                         if (mask & (0x1 << j)) {
1779                                 if (mask != 0xF)
1780                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1781                                 reta |= reta_conf[idx].reta[shift + j] <<
1782                                                 (CHAR_BIT * j);
1783                         }
1784                 }
1785                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1786         }
1787
1788         return 0;
1789 }
1790
1791 static int
1792 fm10k_reta_query(struct rte_eth_dev *dev,
1793                         struct rte_eth_rss_reta_entry64 *reta_conf,
1794                         uint16_t reta_size)
1795 {
1796         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1797         uint16_t i, j, idx, shift;
1798         uint8_t mask;
1799         uint32_t reta;
1800
1801         PMD_INIT_FUNC_TRACE();
1802
1803         if (reta_size < FM10K_MAX_RSS_INDICES) {
1804                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1805                         "(%d) doesn't match the number hardware can supported "
1806                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1807                 return -EINVAL;
1808         }
1809
1810         /*
1811          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1812          * 128-entries in 32 registers
1813          */
1814         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1815                 idx = i / RTE_RETA_GROUP_SIZE;
1816                 shift = i % RTE_RETA_GROUP_SIZE;
1817                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1818                                 BIT_MASK_PER_UINT32);
1819                 if (mask == 0)
1820                         continue;
1821
1822                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1823                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1824                         if (mask & (0x1 << j))
1825                                 reta_conf[idx].reta[shift + j] = ((reta >>
1826                                         CHAR_BIT * j) & UINT8_MAX);
1827                 }
1828         }
1829
1830         return 0;
1831 }
1832
1833 static int
1834 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1835         struct rte_eth_rss_conf *rss_conf)
1836 {
1837         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1838         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1839         uint32_t mrqc;
1840         uint64_t hf = rss_conf->rss_hf;
1841         int i;
1842
1843         PMD_INIT_FUNC_TRACE();
1844
1845         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1846                 FM10K_RSSRK_ENTRIES_PER_REG)
1847                 return -EINVAL;
1848
1849         if (hf == 0)
1850                 return -EINVAL;
1851
1852         mrqc = 0;
1853         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1854         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1855         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1856         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1857         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1858         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1859         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1860         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1861         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1862
1863         /* If the mapping doesn't fit any supported, return */
1864         if (mrqc == 0)
1865                 return -EINVAL;
1866
1867         if (key != NULL)
1868                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1869                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1870
1871         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1872
1873         return 0;
1874 }
1875
1876 static int
1877 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1878         struct rte_eth_rss_conf *rss_conf)
1879 {
1880         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1881         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1882         uint32_t mrqc;
1883         uint64_t hf;
1884         int i;
1885
1886         PMD_INIT_FUNC_TRACE();
1887
1888         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1889                                 FM10K_RSSRK_ENTRIES_PER_REG)
1890                 return -EINVAL;
1891
1892         if (key != NULL)
1893                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1894                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1895
1896         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1897         hf = 0;
1898         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1899         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1900         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1901         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1902         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1903         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1904         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1905         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1906         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1907
1908         rss_conf->rss_hf = hf;
1909
1910         return 0;
1911 }
1912
1913 static void
1914 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1915 {
1916         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1917         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1918
1919         /* Bind all local non-queue interrupt to vector 0 */
1920         int_map |= 0;
1921
1922         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1923         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1924         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1925         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1926         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1927         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1928
1929         /* Enable misc causes */
1930         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1931                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1932                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1933                                 FM10K_EIMR_ENABLE(MAILBOX) |
1934                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1935                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1936                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1937                                 FM10K_EIMR_ENABLE(VFLR));
1938
1939         /* Enable ITR 0 */
1940         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1941                                         FM10K_ITR_MASK_CLEAR);
1942         FM10K_WRITE_FLUSH(hw);
1943 }
1944
1945 static void
1946 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1947 {
1948         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1949         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1950
1951         int_map |= 0;
1952
1953         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1954         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1955         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1956         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1957         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1958         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1959
1960         /* Disable misc causes */
1961         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1962                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1963                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1964                                 FM10K_EIMR_DISABLE(MAILBOX) |
1965                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1966                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1967                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1968                                 FM10K_EIMR_DISABLE(VFLR));
1969
1970         /* Disable ITR 0 */
1971         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1972         FM10K_WRITE_FLUSH(hw);
1973 }
1974
1975 static void
1976 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1977 {
1978         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1979         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1980
1981         /* Bind all local non-queue interrupt to vector 0 */
1982         int_map |= 0;
1983
1984         /* Only INT 0 available, other 15 are reserved. */
1985         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1986
1987         /* Enable ITR 0 */
1988         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1989                                         FM10K_ITR_MASK_CLEAR);
1990         FM10K_WRITE_FLUSH(hw);
1991 }
1992
1993 static void
1994 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1995 {
1996         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1997         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1998
1999         int_map |= 0;
2000
2001         /* Only INT 0 available, other 15 are reserved. */
2002         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2003
2004         /* Disable ITR 0 */
2005         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2006         FM10K_WRITE_FLUSH(hw);
2007 }
2008
2009 static int
2010 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2011 {
2012         struct fm10k_fault fault;
2013         int err;
2014         const char *estr = "Unknown error";
2015
2016         /* Process PCA fault */
2017         if (eicr & FM10K_EICR_PCA_FAULT) {
2018                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2019                 if (err)
2020                         goto error;
2021                 switch (fault.type) {
2022                 case PCA_NO_FAULT:
2023                         estr = "PCA_NO_FAULT"; break;
2024                 case PCA_UNMAPPED_ADDR:
2025                         estr = "PCA_UNMAPPED_ADDR"; break;
2026                 case PCA_BAD_QACCESS_PF:
2027                         estr = "PCA_BAD_QACCESS_PF"; break;
2028                 case PCA_BAD_QACCESS_VF:
2029                         estr = "PCA_BAD_QACCESS_VF"; break;
2030                 case PCA_MALICIOUS_REQ:
2031                         estr = "PCA_MALICIOUS_REQ"; break;
2032                 case PCA_POISONED_TLP:
2033                         estr = "PCA_POISONED_TLP"; break;
2034                 case PCA_TLP_ABORT:
2035                         estr = "PCA_TLP_ABORT"; break;
2036                 default:
2037                         goto error;
2038                 }
2039                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2040                         estr, fault.func ? "VF" : "PF", fault.func,
2041                         fault.address, fault.specinfo);
2042         }
2043
2044         /* Process THI fault */
2045         if (eicr & FM10K_EICR_THI_FAULT) {
2046                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2047                 if (err)
2048                         goto error;
2049                 switch (fault.type) {
2050                 case THI_NO_FAULT:
2051                         estr = "THI_NO_FAULT"; break;
2052                 case THI_MAL_DIS_Q_FAULT:
2053                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2054                 default:
2055                         goto error;
2056                 }
2057                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2058                         estr, fault.func ? "VF" : "PF", fault.func,
2059                         fault.address, fault.specinfo);
2060         }
2061
2062         /* Process FUM fault */
2063         if (eicr & FM10K_EICR_FUM_FAULT) {
2064                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2065                 if (err)
2066                         goto error;
2067                 switch (fault.type) {
2068                 case FUM_NO_FAULT:
2069                         estr = "FUM_NO_FAULT"; break;
2070                 case FUM_UNMAPPED_ADDR:
2071                         estr = "FUM_UNMAPPED_ADDR"; break;
2072                 case FUM_POISONED_TLP:
2073                         estr = "FUM_POISONED_TLP"; break;
2074                 case FUM_BAD_VF_QACCESS:
2075                         estr = "FUM_BAD_VF_QACCESS"; break;
2076                 case FUM_ADD_DECODE_ERR:
2077                         estr = "FUM_ADD_DECODE_ERR"; break;
2078                 case FUM_RO_ERROR:
2079                         estr = "FUM_RO_ERROR"; break;
2080                 case FUM_QPRC_CRC_ERROR:
2081                         estr = "FUM_QPRC_CRC_ERROR"; break;
2082                 case FUM_CSR_TIMEOUT:
2083                         estr = "FUM_CSR_TIMEOUT"; break;
2084                 case FUM_INVALID_TYPE:
2085                         estr = "FUM_INVALID_TYPE"; break;
2086                 case FUM_INVALID_LENGTH:
2087                         estr = "FUM_INVALID_LENGTH"; break;
2088                 case FUM_INVALID_BE:
2089                         estr = "FUM_INVALID_BE"; break;
2090                 case FUM_INVALID_ALIGN:
2091                         estr = "FUM_INVALID_ALIGN"; break;
2092                 default:
2093                         goto error;
2094                 }
2095                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2096                         estr, fault.func ? "VF" : "PF", fault.func,
2097                         fault.address, fault.specinfo);
2098         }
2099
2100         return 0;
2101 error:
2102         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2103         return err;
2104 }
2105
2106 /**
2107  * PF interrupt handler triggered by NIC for handling specific interrupt.
2108  *
2109  * @param handle
2110  *  Pointer to interrupt handle.
2111  * @param param
2112  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2113  *
2114  * @return
2115  *  void
2116  */
2117 static void
2118 fm10k_dev_interrupt_handler_pf(
2119                         __rte_unused struct rte_intr_handle *handle,
2120                         void *param)
2121 {
2122         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2123         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2124         uint32_t cause, status;
2125
2126         if (hw->mac.type != fm10k_mac_pf)
2127                 return;
2128
2129         cause = FM10K_READ_REG(hw, FM10K_EICR);
2130
2131         /* Handle PCI fault cases */
2132         if (cause & FM10K_EICR_FAULT_MASK) {
2133                 PMD_INIT_LOG(ERR, "INT: find fault!");
2134                 fm10k_dev_handle_fault(hw, cause);
2135         }
2136
2137         /* Handle switch up/down */
2138         if (cause & FM10K_EICR_SWITCHNOTREADY)
2139                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2140
2141         if (cause & FM10K_EICR_SWITCHREADY)
2142                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2143
2144         /* Handle mailbox message */
2145         fm10k_mbx_lock(hw);
2146         hw->mbx.ops.process(hw, &hw->mbx);
2147         fm10k_mbx_unlock(hw);
2148
2149         /* Handle SRAM error */
2150         if (cause & FM10K_EICR_SRAMERROR) {
2151                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2152
2153                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2154                 /* Write to clear pending bits */
2155                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2156
2157                 /* Todo: print out error message after shared code  updates */
2158         }
2159
2160         /* Clear these 3 events if having any */
2161         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2162                  FM10K_EICR_SWITCHREADY;
2163         if (cause)
2164                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2165
2166         /* Re-enable interrupt from device side */
2167         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2168                                         FM10K_ITR_MASK_CLEAR);
2169         /* Re-enable interrupt from host side */
2170         rte_intr_enable(&(dev->pci_dev->intr_handle));
2171 }
2172
2173 /**
2174  * VF interrupt handler triggered by NIC for handling specific interrupt.
2175  *
2176  * @param handle
2177  *  Pointer to interrupt handle.
2178  * @param param
2179  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2180  *
2181  * @return
2182  *  void
2183  */
2184 static void
2185 fm10k_dev_interrupt_handler_vf(
2186                         __rte_unused struct rte_intr_handle *handle,
2187                         void *param)
2188 {
2189         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2190         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2191
2192         if (hw->mac.type != fm10k_mac_vf)
2193                 return;
2194
2195         /* Handle mailbox message if lock is acquired */
2196         fm10k_mbx_lock(hw);
2197         hw->mbx.ops.process(hw, &hw->mbx);
2198         fm10k_mbx_unlock(hw);
2199
2200         /* Re-enable interrupt from device side */
2201         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2202                                         FM10K_ITR_MASK_CLEAR);
2203         /* Re-enable interrupt from host side */
2204         rte_intr_enable(&(dev->pci_dev->intr_handle));
2205 }
2206
2207 /* Mailbox message handler in VF */
2208 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2209         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2210         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2211         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2212         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2213 };
2214
2215 /* Mailbox message handler in PF */
2216 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2217         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2218         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2219         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2220         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2221         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2222         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2223         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2224 };
2225
2226 static int
2227 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2228 {
2229         int err;
2230
2231         /* Initialize mailbox lock */
2232         fm10k_mbx_initlock(hw);
2233
2234         /* Replace default message handler with new ones */
2235         if (hw->mac.type == fm10k_mac_pf)
2236                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2237         else
2238                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2239
2240         if (err) {
2241                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2242                                 err);
2243                 return err;
2244         }
2245         /* Connect to SM for PF device or PF for VF device */
2246         return hw->mbx.ops.connect(hw, &hw->mbx);
2247 }
2248
2249 static void
2250 fm10k_close_mbx_service(struct fm10k_hw *hw)
2251 {
2252         /* Disconnect from SM for PF device or PF for VF device */
2253         hw->mbx.ops.disconnect(hw, &hw->mbx);
2254 }
2255
2256 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2257         .dev_configure          = fm10k_dev_configure,
2258         .dev_start              = fm10k_dev_start,
2259         .dev_stop               = fm10k_dev_stop,
2260         .dev_close              = fm10k_dev_close,
2261         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2262         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2263         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2264         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2265         .stats_get              = fm10k_stats_get,
2266         .stats_reset            = fm10k_stats_reset,
2267         .link_update            = fm10k_link_update,
2268         .dev_infos_get          = fm10k_dev_infos_get,
2269         .vlan_filter_set        = fm10k_vlan_filter_set,
2270         .vlan_offload_set       = fm10k_vlan_offload_set,
2271         .mac_addr_add           = fm10k_macaddr_add,
2272         .mac_addr_remove        = fm10k_macaddr_remove,
2273         .rx_queue_start         = fm10k_dev_rx_queue_start,
2274         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2275         .tx_queue_start         = fm10k_dev_tx_queue_start,
2276         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2277         .rx_queue_setup         = fm10k_rx_queue_setup,
2278         .rx_queue_release       = fm10k_rx_queue_release,
2279         .tx_queue_setup         = fm10k_tx_queue_setup,
2280         .tx_queue_release       = fm10k_tx_queue_release,
2281         .reta_update            = fm10k_reta_update,
2282         .reta_query             = fm10k_reta_query,
2283         .rss_hash_update        = fm10k_rss_hash_update,
2284         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2285 };
2286
2287 static void
2288 fm10k_params_init(struct rte_eth_dev *dev)
2289 {
2290         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2291         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2292
2293         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2294          * there is no way to get link status without reading BAR4.  Until this
2295          * works, assume we have maximum bandwidth.
2296          * @todo - fix bus info
2297          */
2298         hw->bus_caps.speed = fm10k_bus_speed_8000;
2299         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2300         hw->bus_caps.payload = fm10k_bus_payload_512;
2301         hw->bus.speed = fm10k_bus_speed_8000;
2302         hw->bus.width = fm10k_bus_width_pcie_x8;
2303         hw->bus.payload = fm10k_bus_payload_256;
2304
2305         info->rx_vec_allowed = true;
2306 }
2307
2308 static int
2309 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2310 {
2311         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2312         int diag;
2313         struct fm10k_macvlan_filter_info *macvlan;
2314
2315         PMD_INIT_FUNC_TRACE();
2316
2317         dev->dev_ops = &fm10k_eth_dev_ops;
2318         dev->rx_pkt_burst = &fm10k_recv_pkts;
2319         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2320
2321         if (dev->data->scattered_rx)
2322                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2323
2324         /* only initialize in the primary process */
2325         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2326                 return 0;
2327
2328         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2329         memset(macvlan, 0, sizeof(*macvlan));
2330         /* Vendor and Device ID need to be set before init of shared code */
2331         memset(hw, 0, sizeof(*hw));
2332         hw->device_id = dev->pci_dev->id.device_id;
2333         hw->vendor_id = dev->pci_dev->id.vendor_id;
2334         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2335         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2336         hw->revision_id = 0;
2337         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2338         if (hw->hw_addr == NULL) {
2339                 PMD_INIT_LOG(ERR, "Bad mem resource."
2340                         " Try to blacklist unused devices.");
2341                 return -EIO;
2342         }
2343
2344         /* Store fm10k_adapter pointer */
2345         hw->back = dev->data->dev_private;
2346
2347         /* Initialize the shared code */
2348         diag = fm10k_init_shared_code(hw);
2349         if (diag != FM10K_SUCCESS) {
2350                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2351                 return -EIO;
2352         }
2353
2354         /* Initialize parameters */
2355         fm10k_params_init(dev);
2356
2357         /* Initialize the hw */
2358         diag = fm10k_init_hw(hw);
2359         if (diag != FM10K_SUCCESS) {
2360                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2361                 return -EIO;
2362         }
2363
2364         /* Initialize MAC address(es) */
2365         dev->data->mac_addrs = rte_zmalloc("fm10k",
2366                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2367         if (dev->data->mac_addrs == NULL) {
2368                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2369                 return -ENOMEM;
2370         }
2371
2372         diag = fm10k_read_mac_addr(hw);
2373
2374         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2375                         &dev->data->mac_addrs[0]);
2376
2377         if (diag != FM10K_SUCCESS ||
2378                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2379
2380                 /* Generate a random addr */
2381                 eth_random_addr(hw->mac.addr);
2382                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2383                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2384                 &dev->data->mac_addrs[0]);
2385         }
2386
2387         /* Reset the hw statistics */
2388         fm10k_stats_reset(dev);
2389
2390         /* Reset the hw */
2391         diag = fm10k_reset_hw(hw);
2392         if (diag != FM10K_SUCCESS) {
2393                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2394                 return -EIO;
2395         }
2396
2397         /* Setup mailbox service */
2398         diag = fm10k_setup_mbx_service(hw);
2399         if (diag != FM10K_SUCCESS) {
2400                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2401                 return -EIO;
2402         }
2403
2404         /*PF/VF has different interrupt handling mechanism */
2405         if (hw->mac.type == fm10k_mac_pf) {
2406                 /* register callback func to eal lib */
2407                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2408                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2409
2410                 /* enable MISC interrupt */
2411                 fm10k_dev_enable_intr_pf(dev);
2412         } else { /* VF */
2413                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2414                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2415
2416                 fm10k_dev_enable_intr_vf(dev);
2417         }
2418
2419         /* Enable uio intr after callback registered */
2420         rte_intr_enable(&(dev->pci_dev->intr_handle));
2421
2422         hw->mac.ops.update_int_moderator(hw);
2423
2424         /* Make sure Switch Manager is ready before going forward. */
2425         if (hw->mac.type == fm10k_mac_pf) {
2426                 int switch_ready = 0;
2427                 int i;
2428
2429                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2430                         fm10k_mbx_lock(hw);
2431                         hw->mac.ops.get_host_state(hw, &switch_ready);
2432                         fm10k_mbx_unlock(hw);
2433                         if (switch_ready)
2434                                 break;
2435                         /* Delay some time to acquire async LPORT_MAP info. */
2436                         rte_delay_us(WAIT_SWITCH_MSG_US);
2437                 }
2438
2439                 if (switch_ready == 0) {
2440                         PMD_INIT_LOG(ERR, "switch is not ready");
2441                         return -1;
2442                 }
2443         }
2444
2445         /*
2446          * Below function will trigger operations on mailbox, acquire lock to
2447          * avoid race condition from interrupt handler. Operations on mailbox
2448          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2449          * will handle and generate an interrupt to our side. Then,  FIFO in
2450          * mailbox will be touched.
2451          */
2452         fm10k_mbx_lock(hw);
2453         /* Enable port first */
2454         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2455
2456         /* Set unicast mode by default. App can change to other mode in other
2457          * API func.
2458          */
2459         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2460                                         FM10K_XCAST_MODE_NONE);
2461
2462         fm10k_mbx_unlock(hw);
2463
2464         /* Add default mac address */
2465         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2466                 MAIN_VSI_POOL_NUMBER);
2467
2468         return 0;
2469 }
2470
2471 static int
2472 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2473 {
2474         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2475
2476         PMD_INIT_FUNC_TRACE();
2477
2478         /* only uninitialize in the primary process */
2479         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2480                 return 0;
2481
2482         /* safe to close dev here */
2483         fm10k_dev_close(dev);
2484
2485         dev->dev_ops = NULL;
2486         dev->rx_pkt_burst = NULL;
2487         dev->tx_pkt_burst = NULL;
2488
2489         /* disable uio/vfio intr */
2490         rte_intr_disable(&(dev->pci_dev->intr_handle));
2491
2492         /*PF/VF has different interrupt handling mechanism */
2493         if (hw->mac.type == fm10k_mac_pf) {
2494                 /* disable interrupt */
2495                 fm10k_dev_disable_intr_pf(dev);
2496
2497                 /* unregister callback func to eal lib */
2498                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2499                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2500         } else {
2501                 /* disable interrupt */
2502                 fm10k_dev_disable_intr_vf(dev);
2503
2504                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2505                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2506         }
2507
2508         /* free mac memory */
2509         if (dev->data->mac_addrs) {
2510                 rte_free(dev->data->mac_addrs);
2511                 dev->data->mac_addrs = NULL;
2512         }
2513
2514         memset(hw, 0, sizeof(*hw));
2515
2516         return 0;
2517 }
2518
2519 /*
2520  * The set of PCI devices this driver supports. This driver will enable both PF
2521  * and SRIOV-VF devices.
2522  */
2523 static const struct rte_pci_id pci_id_fm10k_map[] = {
2524 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2525 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2526 #include "rte_pci_dev_ids.h"
2527         { .vendor_id = 0, /* sentinel */ },
2528 };
2529
2530 static struct eth_driver rte_pmd_fm10k = {
2531         .pci_drv = {
2532                 .name = "rte_pmd_fm10k",
2533                 .id_table = pci_id_fm10k_map,
2534                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2535         },
2536         .eth_dev_init = eth_fm10k_dev_init,
2537         .eth_dev_uninit = eth_fm10k_dev_uninit,
2538         .dev_private_size = sizeof(struct fm10k_adapter),
2539 };
2540
2541 /*
2542  * Driver initialization routine.
2543  * Invoked once at EAL init time.
2544  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2545  */
2546 static int
2547 rte_pmd_fm10k_init(__rte_unused const char *name,
2548         __rte_unused const char *params)
2549 {
2550         PMD_INIT_FUNC_TRACE();
2551         rte_eth_driver_register(&rte_pmd_fm10k);
2552         return 0;
2553 }
2554
2555 static struct rte_driver rte_fm10k_driver = {
2556         .type = PMD_PDEV,
2557         .init = rte_pmd_fm10k_init,
2558 };
2559
2560 PMD_REGISTER_DRIVER(rte_fm10k_driver);