drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  64 static int
  65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  67         const u8 *mac, bool add, uint32_t pool);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  71
  72 static void
  73 fm10k_mbx_initlock(struct fm10k_hw *hw)
  74 {
  75         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  76 }
  77
  78 static void
  79 fm10k_mbx_lock(struct fm10k_hw *hw)
  80 {
  81         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  82                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  83 }
  84
  85 static void
  86 fm10k_mbx_unlock(struct fm10k_hw *hw)
  87 {
  88         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  89 }
  90
  91 /*
  92  * reset queue to initial state, allocate software buffers used when starting
  93  * device.
  94  * return 0 on success
  95  * return -ENOMEM if buffers cannot be allocated
  96  * return -EINVAL if buffers do not satisfy alignment condition
  97  */
  98 static inline int
  99 rx_queue_reset(struct fm10k_rx_queue *q)
 100 {
 101         uint64_t dma_addr;
 102         int i, diag;
 103         PMD_INIT_FUNC_TRACE();
 104
 105         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 106         if (diag != 0)
 107                 return -ENOMEM;
 108
 109         for (i = 0; i < q->nb_desc; ++i) {
 110                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 111                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 112                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 113                                                 q->nb_desc);
 114                         return -EINVAL;
 115                 }
 116                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 117                 q->hw_ring[i].q.pkt_addr = dma_addr;
 118                 q->hw_ring[i].q.hdr_addr = dma_addr;
 119         }
 120
 121         q->next_dd = 0;
 122         q->next_alloc = 0;
 123         q->next_trigger = q->alloc_thresh - 1;
 124         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 125         q->rxrearm_start = 0;
 126         q->rxrearm_nb = 0;
 127
 128         return 0;
 129 }
 130
 131 /*
 132  * clean queue, descriptor rings, free software buffers used when stopping
 133  * device.
 134  */
 135 static inline void
 136 rx_queue_clean(struct fm10k_rx_queue *q)
 137 {
 138         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 139         uint32_t i;
 140         PMD_INIT_FUNC_TRACE();
 141
 142         /* zero descriptor rings */
 143         for (i = 0; i < q->nb_desc; ++i)
 144                 q->hw_ring[i] = zero;
 145
 146         /* free software buffers */
 147         for (i = 0; i < q->nb_desc; ++i) {
 148                 if (q->sw_ring[i]) {
 149                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 150                         q->sw_ring[i] = NULL;
 151                 }
 152         }
 153 }
 154
 155 /*
 156  * free all queue memory used when releasing the queue (i.e. configure)
 157  */
 158 static inline void
 159 rx_queue_free(struct fm10k_rx_queue *q)
 160 {
 161         PMD_INIT_FUNC_TRACE();
 162         if (q) {
 163                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 164                 rx_queue_clean(q);
 165                 if (q->sw_ring) {
 166                         rte_free(q->sw_ring);
 167                         q->sw_ring = NULL;
 168                 }
 169                 rte_free(q);
 170                 q = NULL;
 171         }
 172 }
 173
 174 /*
 175  * disable RX queue, wait unitl HW finished necessary flush operation
 176  */
 177 static inline int
 178 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 179 {
 180         uint32_t reg, i;
 181
 182         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 183         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 184                         reg & ~FM10K_RXQCTL_ENABLE);
 185
 186         /* Wait 100us at most */
 187         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 188                 rte_delay_us(1);
 189                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 190                 if (!(reg & FM10K_RXQCTL_ENABLE))
 191                         break;
 192         }
 193
 194         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 195                 return -1;
 196
 197         return 0;
 198 }
 199
 200 /*
 201  * reset queue to initial state, allocate software buffers used when starting
 202  * device
 203  */
 204 static inline void
 205 tx_queue_reset(struct fm10k_tx_queue *q)
 206 {
 207         PMD_INIT_FUNC_TRACE();
 208         q->last_free = 0;
 209         q->next_free = 0;
 210         q->nb_used = 0;
 211         q->nb_free = q->nb_desc - 1;
 212         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 213         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 214 }
 215
 216 /*
 217  * clean queue, descriptor rings, free software buffers used when stopping
 218  * device
 219  */
 220 static inline void
 221 tx_queue_clean(struct fm10k_tx_queue *q)
 222 {
 223         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 224         uint32_t i;
 225         PMD_INIT_FUNC_TRACE();
 226
 227         /* zero descriptor rings */
 228         for (i = 0; i < q->nb_desc; ++i)
 229                 q->hw_ring[i] = zero;
 230
 231         /* free software buffers */
 232         for (i = 0; i < q->nb_desc; ++i) {
 233                 if (q->sw_ring[i]) {
 234                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 235                         q->sw_ring[i] = NULL;
 236                 }
 237         }
 238 }
 239
 240 /*
 241  * free all queue memory used when releasing the queue (i.e. configure)
 242  */
 243 static inline void
 244 tx_queue_free(struct fm10k_tx_queue *q)
 245 {
 246         PMD_INIT_FUNC_TRACE();
 247         if (q) {
 248                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 249                 tx_queue_clean(q);
 250                 if (q->rs_tracker.list) {
 251                         rte_free(q->rs_tracker.list);
 252                         q->rs_tracker.list = NULL;
 253                 }
 254                 if (q->sw_ring) {
 255                         rte_free(q->sw_ring);
 256                         q->sw_ring = NULL;
 257                 }
 258                 rte_free(q);
 259                 q = NULL;
 260         }
 261 }
 262
 263 /*
 264  * disable TX queue, wait unitl HW finished necessary flush operation
 265  */
 266 static inline int
 267 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 268 {
 269         uint32_t reg, i;
 270
 271         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 272         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 273                         reg & ~FM10K_TXDCTL_ENABLE);
 274
 275         /* Wait 100us at most */
 276         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 277                 rte_delay_us(1);
 278                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 279                 if (!(reg & FM10K_TXDCTL_ENABLE))
 280                         break;
 281         }
 282
 283         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 284                 return -1;
 285
 286         return 0;
 287 }
 288
 289 static int
 290 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 291 {
 292         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 293         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 294         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 295         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 296
 297         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 298
 299         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 300                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 301                 return -EINVAL;
 302         }
 303
 304         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 305                 return 0;
 306
 307         if (hw->mac.type == fm10k_mac_vf) {
 308                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 309                 return -EINVAL;
 310         }
 311
 312         /* Check VMDQ queue pool number */
 313         if (vmdq_conf->nb_queue_pools >
 314                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 315                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 316                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 317                         vmdq_conf->nb_queue_pools);
 318                 return -EINVAL;
 319         }
 320
 321         return 0;
 322 }
 323
 324 static int
 325 fm10k_dev_configure(struct rte_eth_dev *dev)
 326 {
 327         int ret;
 328
 329         PMD_INIT_FUNC_TRACE();
 330
 331         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 332                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 333         /* multipe queue mode checking */
 334         ret  = fm10k_check_mq_mode(dev);
 335         if (ret != 0) {
 336                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 337                             ret);
 338                 return ret;
 339         }
 340
 341         return 0;
 342 }
 343
 344 /* fls = find last set bit = 32 minus the number of leading zeros */
 345 #ifndef fls
 346 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 347 #endif
 348
 349 static void
 350 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 351 {
 352         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 353         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 354         uint32_t i;
 355
 356         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 357
 358         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 359                 if (!vmdq_conf->pool_map[i].pools)
 360                         continue;
 361                 fm10k_mbx_lock(hw);
 362                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 363                 fm10k_mbx_unlock(hw);
 364         }
 365 }
 366
 367 static void
 368 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 369 {
 370         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 371
 372         /* Add default mac address */
 373         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 374                 MAIN_VSI_POOL_NUMBER);
 375 }
 376
 377 static void
 378 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 379 {
 380         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 381         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 382         uint32_t mrqc, *key, i, reta, j;
 383         uint64_t hf;
 384
 385 #define RSS_KEY_SIZE 40
 386         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 387                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 388                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 389                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 390                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 391                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 392         };
 393
 394         if (dev->data->nb_rx_queues == 1 ||
 395             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 396             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 397                 return;
 398
 399         /* random key is rss_intel_key (default) or user provided (rss_key) */
 400         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 401                 key = (uint32_t *)rss_intel_key;
 402         else
 403                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 404
 405         /* Now fill our hash function seeds, 4 bytes at a time */
 406         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 407                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 408
 409         /*
 410          * Fill in redirection table
 411          * The byte-swap is needed because NIC registers are in
 412          * little-endian order.
 413          */
 414         reta = 0;
 415         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 416                 if (j == dev->data->nb_rx_queues)
 417                         j = 0;
 418                 reta = (reta << CHAR_BIT) | j;
 419                 if ((i & 3) == 3)
 420                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 421                                         rte_bswap32(reta));
 422         }
 423
 424         /*
 425          * Generate RSS hash based on packet types, TCP/UDP
 426          * port numbers and/or IPv4/v6 src and dst addresses
 427          */
 428         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 429         mrqc = 0;
 430         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 431         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 432         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 433         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 434         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 435         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 436         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 437         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 438         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 439
 440         if (mrqc == 0) {
 441                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 442                         "supported", hf);
 443                 return;
 444         }
 445
 446         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 447 }
 448
 449 static void
 450 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 451         uint16_t nb_lport_old, uint16_t nb_lport_new)
 452 {
 453         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 454         uint32_t i;
 455
 456         fm10k_mbx_lock(hw);
 457         /* Disable previous logic ports */
 458         if (nb_lport_old)
 459                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 460                         nb_lport_old, false);
 461         /* Enable new logic ports */
 462         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 463                 nb_lport_new, true);
 464         fm10k_mbx_unlock(hw);
 465
 466         for (i = 0; i < nb_lport_new; i++) {
 467                 /* Set unicast mode by default. App can change
 468                  * to other mode in other API func.
 469                  */
 470                 fm10k_mbx_lock(hw);
 471                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 472                         FM10K_XCAST_MODE_NONE);
 473                 fm10k_mbx_unlock(hw);
 474         }
 475 }
 476
 477 static void
 478 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 479 {
 480         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 481         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 482         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 483         struct fm10k_macvlan_filter_info *macvlan;
 484         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 485         uint16_t nb_lport_new, nb_lport_old;
 486
 487         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 488         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 489
 490         fm10k_dev_rss_configure(dev);
 491
 492         /* only PF supports VMDQ */
 493         if (hw->mac.type != fm10k_mac_pf)
 494                 return;
 495
 496         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 497                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 498
 499         /* no pool number change, no need to update logic port and VLAN/MAC */
 500         if (macvlan->nb_queue_pools == nb_queue_pools)
 501                 return;
 502
 503         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 504         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 505         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 506
 507         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 508         memset(dev->data->mac_addrs, 0,
 509                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 510         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 511                 &dev->data->mac_addrs[0]);
 512         memset(macvlan, 0, sizeof(*macvlan));
 513         macvlan->nb_queue_pools = nb_queue_pools;
 514
 515         if (nb_queue_pools)
 516                 fm10k_dev_vmdq_rx_configure(dev);
 517         else
 518                 fm10k_dev_pf_main_vsi_reset(dev);
 519 }
 520
 521 static int
 522 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 523 {
 524         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 525         int i, ret;
 526         struct fm10k_tx_queue *txq;
 527         uint64_t base_addr;
 528         uint32_t size;
 529
 530         /* Disable TXINT to avoid possible interrupt */
 531         for (i = 0; i < hw->mac.max_queues; i++)
 532                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 533                                 3 << FM10K_TXINT_TIMER_SHIFT);
 534
 535         /* Setup TX queue */
 536         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 537                 txq = dev->data->tx_queues[i];
 538                 base_addr = txq->hw_ring_phys_addr;
 539                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 540
 541                 /* disable queue to avoid issues while updating state */
 542                 ret = tx_queue_disable(hw, i);
 543                 if (ret) {
 544                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 545                         return -1;
 546                 }
 547
 548                 /* set location and size for descriptor ring */
 549                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 550                                 base_addr & UINT64_LOWER_32BITS_MASK);
 551                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 552                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 553                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 554         }
 555         return 0;
 556 }
 557
 558 static int
 559 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 560 {
 561         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 562         int i, ret;
 563         struct fm10k_rx_queue *rxq;
 564         uint64_t base_addr;
 565         uint32_t size;
 566         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 567         uint16_t buf_size;
 568
 569         /* Disable RXINT to avoid possible interrupt */
 570         for (i = 0; i < hw->mac.max_queues; i++)
 571                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 572                                 3 << FM10K_RXINT_TIMER_SHIFT);
 573
 574         /* Setup RX queues */
 575         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 576                 rxq = dev->data->rx_queues[i];
 577                 base_addr = rxq->hw_ring_phys_addr;
 578                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 579
 580                 /* disable queue to avoid issues while updating state */
 581                 ret = rx_queue_disable(hw, i);
 582                 if (ret) {
 583                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 584                         return -1;
 585                 }
 586
 587                 /* Setup the Base and Length of the Rx Descriptor Ring */
 588                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 589                                 base_addr & UINT64_LOWER_32BITS_MASK);
 590                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 591                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 592                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 593
 594                 /* Configure the Rx buffer size for one buff without split */
 595                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 596                         RTE_PKTMBUF_HEADROOM);
 597                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 598                  * reserved for this purpose, and the worst case could be 511B.
 599                  * But SRR reg assumes all buffers have the same size. In order
 600                  * to fill the gap, we'll have to consider the worst case and
 601                  * assume 512B is reserved. If we don't do so, it's possible
 602                  * for HW to overwrite data to next mbuf.
 603                  */
 604                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 605
 606                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 607                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 608
 609                 /* It adds dual VLAN length for supporting dual VLAN */
 610                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 611                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 612                         dev->data->dev_conf.rxmode.enable_scatter) {
 613                         uint32_t reg;
 614                         dev->data->scattered_rx = 1;
 615                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 616                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 617                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 618                 }
 619
 620                 /* Enable drop on empty, it's RO for VF */
 621                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 622                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 623
 624                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 625                 FM10K_WRITE_FLUSH(hw);
 626         }
 627
 628         /* Configure VMDQ/RSS if applicable */
 629         fm10k_dev_mq_rx_configure(dev);
 630
 631         /* Decide the best RX function */
 632         fm10k_set_rx_function(dev);
 633
 634         return 0;
 635 }
 636
 637 static int
 638 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 639 {
 640         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 641         int err = -1;
 642         uint32_t reg;
 643         struct fm10k_rx_queue *rxq;
 644
 645         PMD_INIT_FUNC_TRACE();
 646
 647         if (rx_queue_id < dev->data->nb_rx_queues) {
 648                 rxq = dev->data->rx_queues[rx_queue_id];
 649                 err = rx_queue_reset(rxq);
 650                 if (err == -ENOMEM) {
 651                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 652                         return err;
 653                 } else if (err == -EINVAL) {
 654                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 655                                 " %d", err);
 656                         return err;
 657                 }
 658
 659                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 660                  * Note: this must be done AFTER the queue is enabled on real
 661                  * hardware, but BEFORE the queue is enabled when using the
 662                  * emulation platform. Do it in both places for now and remove
 663                  * this comment and the following two register writes when the
 664                  * emulation platform is no longer being used.
 665                  */
 666                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 667                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 668
 669                 /* Set PF ownership flag for PF devices */
 670                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 671                 if (hw->mac.type == fm10k_mac_pf)
 672                         reg |= FM10K_RXQCTL_PF;
 673                 reg |= FM10K_RXQCTL_ENABLE;
 674                 /* enable RX queue */
 675                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 676                 FM10K_WRITE_FLUSH(hw);
 677
 678                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 679                  * Note: this must be done AFTER the queue is enabled
 680                  */
 681                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 682                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 683         }
 684
 685         return err;
 686 }
 687
 688 static int
 689 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 690 {
 691         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 692
 693         PMD_INIT_FUNC_TRACE();
 694
 695         if (rx_queue_id < dev->data->nb_rx_queues) {
 696                 /* Disable RX queue */
 697                 rx_queue_disable(hw, rx_queue_id);
 698
 699                 /* Free mbuf and clean HW ring */
 700                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 701         }
 702
 703         return 0;
 704 }
 705
 706 static int
 707 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 708 {
 709         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 710         /** @todo - this should be defined in the shared code */
 711 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 712         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 713         int err = 0;
 714
 715         PMD_INIT_FUNC_TRACE();
 716
 717         if (tx_queue_id < dev->data->nb_tx_queues) {
 718                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 719
 720                 /* reset head and tail pointers */
 721                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 722                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 723
 724                 /* enable TX queue */
 725                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 726                                         FM10K_TXDCTL_ENABLE | txdctl);
 727                 FM10K_WRITE_FLUSH(hw);
 728         } else
 729                 err = -1;
 730
 731         return err;
 732 }
 733
 734 static int
 735 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 736 {
 737         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 738
 739         PMD_INIT_FUNC_TRACE();
 740
 741         if (tx_queue_id < dev->data->nb_tx_queues) {
 742                 tx_queue_disable(hw, tx_queue_id);
 743                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 744         }
 745
 746         return 0;
 747 }
 748
 749 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 750 {
 751         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 752                 != FM10K_DGLORTMAP_NONE);
 753 }
 754
 755 static void
 756 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 757 {
 758         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 759         int status;
 760
 761         PMD_INIT_FUNC_TRACE();
 762
 763         /* Return if it didn't acquire valid glort range */
 764         if (!fm10k_glort_valid(hw))
 765                 return;
 766
 767         fm10k_mbx_lock(hw);
 768         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 769                                 FM10K_XCAST_MODE_PROMISC);
 770         fm10k_mbx_unlock(hw);
 771
 772         if (status != FM10K_SUCCESS)
 773                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 774 }
 775
 776 static void
 777 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 778 {
 779         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 780         uint8_t mode;
 781         int status;
 782
 783         PMD_INIT_FUNC_TRACE();
 784
 785         /* Return if it didn't acquire valid glort range */
 786         if (!fm10k_glort_valid(hw))
 787                 return;
 788
 789         if (dev->data->all_multicast == 1)
 790                 mode = FM10K_XCAST_MODE_ALLMULTI;
 791         else
 792                 mode = FM10K_XCAST_MODE_NONE;
 793
 794         fm10k_mbx_lock(hw);
 795         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 796                                 mode);
 797         fm10k_mbx_unlock(hw);
 798
 799         if (status != FM10K_SUCCESS)
 800                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 801 }
 802
 803 static void
 804 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 805 {
 806         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 807         int status;
 808
 809         PMD_INIT_FUNC_TRACE();
 810
 811         /* Return if it didn't acquire valid glort range */
 812         if (!fm10k_glort_valid(hw))
 813                 return;
 814
 815         /* If promiscuous mode is enabled, it doesn't make sense to enable
 816          * allmulticast and disable promiscuous since fm10k only can select
 817          * one of the modes.
 818          */
 819         if (dev->data->promiscuous) {
 820                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 821                         "needn't enable allmulticast");
 822                 return;
 823         }
 824
 825         fm10k_mbx_lock(hw);
 826         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 827                                 FM10K_XCAST_MODE_ALLMULTI);
 828         fm10k_mbx_unlock(hw);
 829
 830         if (status != FM10K_SUCCESS)
 831                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 832 }
 833
 834 static void
 835 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 836 {
 837         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 838         int status;
 839
 840         PMD_INIT_FUNC_TRACE();
 841
 842         /* Return if it didn't acquire valid glort range */
 843         if (!fm10k_glort_valid(hw))
 844                 return;
 845
 846         if (dev->data->promiscuous) {
 847                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 848                         "since promisc mode is enabled");
 849                 return;
 850         }
 851
 852         fm10k_mbx_lock(hw);
 853         /* Change mode to unicast mode */
 854         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 855                                 FM10K_XCAST_MODE_NONE);
 856         fm10k_mbx_unlock(hw);
 857
 858         if (status != FM10K_SUCCESS)
 859                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 860 }
 861
 862 static void
 863 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 864 {
 865         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 866         uint32_t dglortdec, pool_len, rss_len, i;
 867         uint16_t nb_queue_pools;
 868         struct fm10k_macvlan_filter_info *macvlan;
 869
 870         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 871         nb_queue_pools = macvlan->nb_queue_pools;
 872         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 873         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 874         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 875
 876         /* Establish only MAP 0 as valid */
 877         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 878
 879         /* Configure VMDQ/RSS DGlort Decoder */
 880         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 881
 882         /* Invalidate all other GLORT entries */
 883         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 884                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 885                                 FM10K_DGLORTMAP_NONE);
 886 }
 887
 888 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 889 static int
 890 fm10k_dev_start(struct rte_eth_dev *dev)
 891 {
 892         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 893         int i, diag;
 894
 895         PMD_INIT_FUNC_TRACE();
 896
 897         /* stop, init, then start the hw */
 898         diag = fm10k_stop_hw(hw);
 899         if (diag != FM10K_SUCCESS) {
 900                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 901                 return -EIO;
 902         }
 903
 904         diag = fm10k_init_hw(hw);
 905         if (diag != FM10K_SUCCESS) {
 906                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 907                 return -EIO;
 908         }
 909
 910         diag = fm10k_start_hw(hw);
 911         if (diag != FM10K_SUCCESS) {
 912                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 913                 return -EIO;
 914         }
 915
 916         diag = fm10k_dev_tx_init(dev);
 917         if (diag) {
 918                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 919                 return diag;
 920         }
 921
 922         diag = fm10k_dev_rx_init(dev);
 923         if (diag) {
 924                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 925                 return diag;
 926         }
 927
 928         if (hw->mac.type == fm10k_mac_pf)
 929                 fm10k_dev_dglort_map_configure(dev);
 930
 931         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 932                 struct fm10k_rx_queue *rxq;
 933                 rxq = dev->data->rx_queues[i];
 934
 935                 if (rxq->rx_deferred_start)
 936                         continue;
 937                 diag = fm10k_dev_rx_queue_start(dev, i);
 938                 if (diag != 0) {
 939                         int j;
 940                         for (j = 0; j < i; ++j)
 941                                 rx_queue_clean(dev->data->rx_queues[j]);
 942                         return diag;
 943                 }
 944         }
 945
 946         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 947                 struct fm10k_tx_queue *txq;
 948                 txq = dev->data->tx_queues[i];
 949
 950                 if (txq->tx_deferred_start)
 951                         continue;
 952                 diag = fm10k_dev_tx_queue_start(dev, i);
 953                 if (diag != 0) {
 954                         int j;
 955                         for (j = 0; j < i; ++j)
 956                                 tx_queue_clean(dev->data->tx_queues[j]);
 957                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 958                                 rx_queue_clean(dev->data->rx_queues[j]);
 959                         return diag;
 960                 }
 961         }
 962
 963         /* Update default vlan when not in VMDQ mode */
 964         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 965                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 966
 967         return 0;
 968 }
 969
 970 static void
 971 fm10k_dev_stop(struct rte_eth_dev *dev)
 972 {
 973         int i;
 974
 975         PMD_INIT_FUNC_TRACE();
 976
 977         if (dev->data->tx_queues)
 978                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 979                         fm10k_dev_tx_queue_stop(dev, i);
 980
 981         if (dev->data->rx_queues)
 982                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 983                         fm10k_dev_rx_queue_stop(dev, i);
 984 }
 985
 986 static void
 987 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 988 {
 989         int i;
 990
 991         PMD_INIT_FUNC_TRACE();
 992
 993         if (dev->data->tx_queues) {
 994                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 995                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 996         }
 997
 998         if (dev->data->rx_queues) {
 999                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1000                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1001         }
1002 }
1003
1004 static void
1005 fm10k_dev_close(struct rte_eth_dev *dev)
1006 {
1007         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1008         uint16_t nb_lport;
1009         struct fm10k_macvlan_filter_info *macvlan;
1010
1011         PMD_INIT_FUNC_TRACE();
1012
1013         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1014         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1015         fm10k_mbx_lock(hw);
1016         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1017                 nb_lport, false);
1018         fm10k_mbx_unlock(hw);
1019
1020         /* Stop mailbox service first */
1021         fm10k_close_mbx_service(hw);
1022         fm10k_dev_stop(dev);
1023         fm10k_dev_queue_release(dev);
1024         fm10k_stop_hw(hw);
1025 }
1026
1027 static int
1028 fm10k_link_update(struct rte_eth_dev *dev,
1029         __rte_unused int wait_to_complete)
1030 {
1031         PMD_INIT_FUNC_TRACE();
1032
1033         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1034          * x8 PCIe interface. For now, we leave the speed undefined since there
1035          * is no 50Gbps Ethernet. */
1036         dev->data->dev_link.link_speed  = 0;
1037         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1038         dev->data->dev_link.link_status = 1;
1039
1040         return 0;
1041 }
1042
1043 static void
1044 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1045 {
1046         uint64_t ipackets, opackets, ibytes, obytes;
1047         struct fm10k_hw *hw =
1048                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1049         struct fm10k_hw_stats *hw_stats =
1050                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1051         int i;
1052
1053         PMD_INIT_FUNC_TRACE();
1054
1055         fm10k_update_hw_stats(hw, hw_stats);
1056
1057         ipackets = opackets = ibytes = obytes = 0;
1058         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1059                 (i < hw->mac.max_queues); ++i) {
1060                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1061                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1062                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1063                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1064                 ipackets += stats->q_ipackets[i];
1065                 opackets += stats->q_opackets[i];
1066                 ibytes   += stats->q_ibytes[i];
1067                 obytes   += stats->q_obytes[i];
1068         }
1069         stats->ipackets = ipackets;
1070         stats->opackets = opackets;
1071         stats->ibytes = ibytes;
1072         stats->obytes = obytes;
1073 }
1074
1075 static void
1076 fm10k_stats_reset(struct rte_eth_dev *dev)
1077 {
1078         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1079         struct fm10k_hw_stats *hw_stats =
1080                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1081
1082         PMD_INIT_FUNC_TRACE();
1083
1084         memset(hw_stats, 0, sizeof(*hw_stats));
1085         fm10k_rebind_hw_stats(hw, hw_stats);
1086 }
1087
1088 static void
1089 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1090         struct rte_eth_dev_info *dev_info)
1091 {
1092         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1093
1094         PMD_INIT_FUNC_TRACE();
1095
1096         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1097         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1098         dev_info->max_rx_queues      = hw->mac.max_queues;
1099         dev_info->max_tx_queues      = hw->mac.max_queues;
1100         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1101         dev_info->max_hash_mac_addrs = 0;
1102         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1103         dev_info->vmdq_pool_base     = 0;
1104         dev_info->vmdq_queue_base    = 0;
1105         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1106         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1107         dev_info->rx_offload_capa =
1108                 DEV_RX_OFFLOAD_VLAN_STRIP |
1109                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1110                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1111                 DEV_RX_OFFLOAD_TCP_CKSUM;
1112         dev_info->tx_offload_capa =
1113                 DEV_TX_OFFLOAD_VLAN_INSERT |
1114                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1115                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1116                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1117                 DEV_TX_OFFLOAD_TCP_TSO;
1118
1119         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1120         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1121
1122         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1123                 .rx_thresh = {
1124                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1125                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1126                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1127                 },
1128                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1129                 .rx_drop_en = 0,
1130         };
1131
1132         dev_info->default_txconf = (struct rte_eth_txconf) {
1133                 .tx_thresh = {
1134                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1135                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1136                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1137                 },
1138                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1139                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1140                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1141                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1142         };
1143
1144         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1145                 .nb_max = FM10K_MAX_RX_DESC,
1146                 .nb_min = FM10K_MIN_RX_DESC,
1147                 .nb_align = FM10K_MULT_RX_DESC,
1148         };
1149
1150         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1151                 .nb_max = FM10K_MAX_TX_DESC,
1152                 .nb_min = FM10K_MIN_TX_DESC,
1153                 .nb_align = FM10K_MULT_TX_DESC,
1154         };
1155 }
1156
1157 static int
1158 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1159 {
1160         s32 result;
1161         uint16_t mac_num = 0;
1162         uint32_t vid_idx, vid_bit, mac_index;
1163         struct fm10k_hw *hw;
1164         struct fm10k_macvlan_filter_info *macvlan;
1165         struct rte_eth_dev_data *data = dev->data;
1166
1167         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1168         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1169
1170         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1171                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1172                 return (-EINVAL);
1173         }
1174
1175         if (vlan_id > ETH_VLAN_ID_MAX) {
1176                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1177                 return (-EINVAL);
1178         }
1179
1180         vid_idx = FM10K_VFTA_IDX(vlan_id);
1181         vid_bit = FM10K_VFTA_BIT(vlan_id);
1182         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1183         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1184                 return 0;
1185         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1186         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1187                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1188                         "in the VLAN filter table");
1189                 return (-EINVAL);
1190         }
1191
1192         fm10k_mbx_lock(hw);
1193         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1194         fm10k_mbx_unlock(hw);
1195         if (result != FM10K_SUCCESS) {
1196                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1197                 return (-EIO);
1198         }
1199
1200         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1201                         (result == FM10K_SUCCESS); mac_index++) {
1202                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1203                         continue;
1204                 if (mac_num > macvlan->mac_num - 1) {
1205                         PMD_INIT_LOG(ERR, "MAC address number "
1206                                         "not match");
1207                         break;
1208                 }
1209                 fm10k_mbx_lock(hw);
1210                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1211                         data->mac_addrs[mac_index].addr_bytes,
1212                         vlan_id, on, 0);
1213                 fm10k_mbx_unlock(hw);
1214                 mac_num++;
1215         }
1216         if (result != FM10K_SUCCESS) {
1217                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1218                 return (-EIO);
1219         }
1220
1221         if (on) {
1222                 macvlan->vlan_num++;
1223                 macvlan->vfta[vid_idx] |= vid_bit;
1224         } else {
1225                 macvlan->vlan_num--;
1226                 macvlan->vfta[vid_idx] &= ~vid_bit;
1227         }
1228         return 0;
1229 }
1230
1231 static void
1232 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1233 {
1234         if (mask & ETH_VLAN_STRIP_MASK) {
1235                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1236                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1237                                         "always on in fm10k");
1238         }
1239
1240         if (mask & ETH_VLAN_EXTEND_MASK) {
1241                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1242                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1243                                         "supported in fm10k");
1244         }
1245
1246         if (mask & ETH_VLAN_FILTER_MASK) {
1247                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1248                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1249         }
1250 }
1251
1252 /* Add/Remove a MAC address, and update filters to main VSI */
1253 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1254                 const u8 *mac, bool add, uint32_t pool)
1255 {
1256         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1257         struct fm10k_macvlan_filter_info *macvlan;
1258         uint32_t i, j, k;
1259
1260         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1261
1262         if (pool != MAIN_VSI_POOL_NUMBER) {
1263                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1264                         "mac to pool %u", pool);
1265                 return;
1266         }
1267         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1268                 if (!macvlan->vfta[j])
1269                         continue;
1270                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1271                         if (!(macvlan->vfta[j] & (1 << k)))
1272                                 continue;
1273                         if (i + 1 > macvlan->vlan_num) {
1274                                 PMD_INIT_LOG(ERR, "vlan number not match");
1275                                 return;
1276                         }
1277                         fm10k_mbx_lock(hw);
1278                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1279                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1280                         fm10k_mbx_unlock(hw);
1281                         i++;
1282                 }
1283         }
1284 }
1285
1286 /* Add/Remove a MAC address, and update filters to VMDQ */
1287 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1288                 const u8 *mac, bool add, uint32_t pool)
1289 {
1290         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1291         struct fm10k_macvlan_filter_info *macvlan;
1292         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1293         uint32_t i;
1294
1295         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1296         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1297
1298         if (pool > macvlan->nb_queue_pools) {
1299                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1300                         " Max pool is %u",
1301                         pool, macvlan->nb_queue_pools);
1302                 return;
1303         }
1304         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1305                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1306                         continue;
1307                 fm10k_mbx_lock(hw);
1308                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1309                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1310                 fm10k_mbx_unlock(hw);
1311         }
1312 }
1313
1314 /* Add/Remove a MAC address, and update filters */
1315 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1316                 const u8 *mac, bool add, uint32_t pool)
1317 {
1318         struct fm10k_macvlan_filter_info *macvlan;
1319
1320         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1321
1322         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1323                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1324         else
1325                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1326
1327         if (add)
1328                 macvlan->mac_num++;
1329         else
1330                 macvlan->mac_num--;
1331 }
1332
1333 /* Add a MAC address, and update filters */
1334 static void
1335 fm10k_macaddr_add(struct rte_eth_dev *dev,
1336                 struct ether_addr *mac_addr,
1337                 uint32_t index,
1338                 uint32_t pool)
1339 {
1340         struct fm10k_macvlan_filter_info *macvlan;
1341
1342         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1343         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1344         macvlan->mac_vmdq_id[index] = pool;
1345 }
1346
1347 /* Remove a MAC address, and update filters */
1348 static void
1349 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1350 {
1351         struct rte_eth_dev_data *data = dev->data;
1352         struct fm10k_macvlan_filter_info *macvlan;
1353
1354         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1355         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1356                         FALSE, macvlan->mac_vmdq_id[index]);
1357         macvlan->mac_vmdq_id[index] = 0;
1358 }
1359
1360 static inline int
1361 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1362 {
1363         if ((request < min) || (request > max) || ((request % mult) != 0))
1364                 return -1;
1365         else
1366                 return 0;
1367 }
1368
1369 /*
1370  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1371  * the physical address is required. If the memzone is already created, then
1372  * this function returns a pointer to the existing memzone.
1373  */
1374 static inline const struct rte_memzone *
1375 allocate_hw_ring(const char *driver_name, const char *ring_name,
1376         uint8_t port_id, uint16_t queue_id, int socket_id,
1377         uint32_t size, uint32_t align)
1378 {
1379         char name[RTE_MEMZONE_NAMESIZE];
1380         const struct rte_memzone *mz;
1381
1382         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1383                  driver_name, ring_name, port_id, queue_id, socket_id);
1384
1385         /* return the memzone if it already exists */
1386         mz = rte_memzone_lookup(name);
1387         if (mz)
1388                 return mz;
1389
1390 #ifdef RTE_LIBRTE_XEN_DOM0
1391         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1392                                            RTE_PGSIZE_2M);
1393 #else
1394         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1395 #endif
1396 }
1397
1398 static inline int
1399 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1400 {
1401         if ((request < min) || (request > max) || ((div % request) != 0))
1402                 return -1;
1403         else
1404                 return 0;
1405 }
1406
1407 static inline int
1408 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1409 {
1410         uint16_t rx_free_thresh;
1411
1412         if (conf->rx_free_thresh == 0)
1413                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1414         else
1415                 rx_free_thresh = conf->rx_free_thresh;
1416
1417         /* make sure the requested threshold satisfies the constraints */
1418         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1419                         FM10K_RX_FREE_THRESH_MAX(q),
1420                         FM10K_RX_FREE_THRESH_DIV(q),
1421                         rx_free_thresh)) {
1422                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1423                         "less than or equal to %u, "
1424                         "greater than or equal to %u, "
1425                         "and a divisor of %u",
1426                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1427                         FM10K_RX_FREE_THRESH_MIN(q),
1428                         FM10K_RX_FREE_THRESH_DIV(q));
1429                 return (-EINVAL);
1430         }
1431
1432         q->alloc_thresh = rx_free_thresh;
1433         q->drop_en = conf->rx_drop_en;
1434         q->rx_deferred_start = conf->rx_deferred_start;
1435
1436         return 0;
1437 }
1438
1439 /*
1440  * Hardware requires specific alignment for Rx packet buffers. At
1441  * least one of the following two conditions must be satisfied.
1442  *  1. Address is 512B aligned
1443  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1444  *
1445  * As such, the driver may need to adjust the DMA address within the
1446  * buffer by up to 512B.
1447  *
1448  * return 1 if the element size is valid, otherwise return 0.
1449  */
1450 static int
1451 mempool_element_size_valid(struct rte_mempool *mp)
1452 {
1453         uint32_t min_size;
1454
1455         /* elt_size includes mbuf header and headroom */
1456         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1457                         RTE_PKTMBUF_HEADROOM;
1458
1459         /* account for up to 512B of alignment */
1460         min_size -= FM10K_RX_DATABUF_ALIGN;
1461
1462         /* sanity check for overflow */
1463         if (min_size > mp->elt_size)
1464                 return 0;
1465
1466         /* size is valid */
1467         return 1;
1468 }
1469
1470 static int
1471 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1472         uint16_t nb_desc, unsigned int socket_id,
1473         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1474 {
1475         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1476         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1477         struct fm10k_rx_queue *q;
1478         const struct rte_memzone *mz;
1479
1480         PMD_INIT_FUNC_TRACE();
1481
1482         /* make sure the mempool element size can account for alignment. */
1483         if (!mempool_element_size_valid(mp)) {
1484                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1485                 return (-EINVAL);
1486         }
1487
1488         /* make sure a valid number of descriptors have been requested */
1489         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1490                                 FM10K_MULT_RX_DESC, nb_desc)) {
1491                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1492                         "less than or equal to %"PRIu32", "
1493                         "greater than or equal to %u, "
1494                         "and a multiple of %u",
1495                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1496                         FM10K_MULT_RX_DESC);
1497                 return (-EINVAL);
1498         }
1499
1500         /*
1501          * if this queue existed already, free the associated memory. The
1502          * queue cannot be reused in case we need to allocate memory on
1503          * different socket than was previously used.
1504          */
1505         if (dev->data->rx_queues[queue_id] != NULL) {
1506                 rx_queue_free(dev->data->rx_queues[queue_id]);
1507                 dev->data->rx_queues[queue_id] = NULL;
1508         }
1509
1510         /* allocate memory for the queue structure */
1511         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1512                                 socket_id);
1513         if (q == NULL) {
1514                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1515                 return (-ENOMEM);
1516         }
1517
1518         /* setup queue */
1519         q->mp = mp;
1520         q->nb_desc = nb_desc;
1521         q->port_id = dev->data->port_id;
1522         q->queue_id = queue_id;
1523         q->tail_ptr = (volatile uint32_t *)
1524                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1525         if (handle_rxconf(q, conf))
1526                 return (-EINVAL);
1527
1528         /* allocate memory for the software ring */
1529         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1530                                         nb_desc * sizeof(struct rte_mbuf *),
1531                                         RTE_CACHE_LINE_SIZE, socket_id);
1532         if (q->sw_ring == NULL) {
1533                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1534                 rte_free(q);
1535                 return (-ENOMEM);
1536         }
1537
1538         /*
1539          * allocate memory for the hardware descriptor ring. A memzone large
1540          * enough to hold the maximum ring size is requested to allow for
1541          * resizing in later calls to the queue setup function.
1542          */
1543         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1544                                 dev->data->port_id, queue_id, socket_id,
1545                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1546         if (mz == NULL) {
1547                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1548                 rte_free(q->sw_ring);
1549                 rte_free(q);
1550                 return (-ENOMEM);
1551         }
1552         q->hw_ring = mz->addr;
1553 #ifdef RTE_LIBRTE_XEN_DOM0
1554         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1555 #else
1556         q->hw_ring_phys_addr = mz->phys_addr;
1557 #endif
1558
1559         /* Check if number of descs satisfied Vector requirement */
1560         if (!rte_is_power_of_2(nb_desc)) {
1561                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1562                                     "preconditions - canceling the feature for "
1563                                     "the whole port[%d]",
1564                              q->queue_id, q->port_id);
1565                 dev_info->rx_vec_allowed = false;
1566         } else
1567                 fm10k_rxq_vec_setup(q);
1568
1569         dev->data->rx_queues[queue_id] = q;
1570         return 0;
1571 }
1572
1573 static void
1574 fm10k_rx_queue_release(void *queue)
1575 {
1576         PMD_INIT_FUNC_TRACE();
1577
1578         rx_queue_free(queue);
1579 }
1580
1581 static inline int
1582 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1583 {
1584         uint16_t tx_free_thresh;
1585         uint16_t tx_rs_thresh;
1586
1587         /* constraint MACROs require that tx_free_thresh is configured
1588          * before tx_rs_thresh */
1589         if (conf->tx_free_thresh == 0)
1590                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1591         else
1592                 tx_free_thresh = conf->tx_free_thresh;
1593
1594         /* make sure the requested threshold satisfies the constraints */
1595         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1596                         FM10K_TX_FREE_THRESH_MAX(q),
1597                         FM10K_TX_FREE_THRESH_DIV(q),
1598                         tx_free_thresh)) {
1599                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1600                         "less than or equal to %u, "
1601                         "greater than or equal to %u, "
1602                         "and a divisor of %u",
1603                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1604                         FM10K_TX_FREE_THRESH_MIN(q),
1605                         FM10K_TX_FREE_THRESH_DIV(q));
1606                 return (-EINVAL);
1607         }
1608
1609         q->free_thresh = tx_free_thresh;
1610
1611         if (conf->tx_rs_thresh == 0)
1612                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1613         else
1614                 tx_rs_thresh = conf->tx_rs_thresh;
1615
1616         q->tx_deferred_start = conf->tx_deferred_start;
1617
1618         /* make sure the requested threshold satisfies the constraints */
1619         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1620                         FM10K_TX_RS_THRESH_MAX(q),
1621                         FM10K_TX_RS_THRESH_DIV(q),
1622                         tx_rs_thresh)) {
1623                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1624                         "less than or equal to %u, "
1625                         "greater than or equal to %u, "
1626                         "and a divisor of %u",
1627                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1628                         FM10K_TX_RS_THRESH_MIN(q),
1629                         FM10K_TX_RS_THRESH_DIV(q));
1630                 return (-EINVAL);
1631         }
1632
1633         q->rs_thresh = tx_rs_thresh;
1634
1635         return 0;
1636 }
1637
1638 static int
1639 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1640         uint16_t nb_desc, unsigned int socket_id,
1641         const struct rte_eth_txconf *conf)
1642 {
1643         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1644         struct fm10k_tx_queue *q;
1645         const struct rte_memzone *mz;
1646
1647         PMD_INIT_FUNC_TRACE();
1648
1649         /* make sure a valid number of descriptors have been requested */
1650         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1651                                 FM10K_MULT_TX_DESC, nb_desc)) {
1652                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1653                         "less than or equal to %"PRIu32", "
1654                         "greater than or equal to %u, "
1655                         "and a multiple of %u",
1656                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1657                         FM10K_MULT_TX_DESC);
1658                 return (-EINVAL);
1659         }
1660
1661         /*
1662          * if this queue existed already, free the associated memory. The
1663          * queue cannot be reused in case we need to allocate memory on
1664          * different socket than was previously used.
1665          */
1666         if (dev->data->tx_queues[queue_id] != NULL) {
1667                 tx_queue_free(dev->data->tx_queues[queue_id]);
1668                 dev->data->tx_queues[queue_id] = NULL;
1669         }
1670
1671         /* allocate memory for the queue structure */
1672         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1673                                 socket_id);
1674         if (q == NULL) {
1675                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1676                 return (-ENOMEM);
1677         }
1678
1679         /* setup queue */
1680         q->nb_desc = nb_desc;
1681         q->port_id = dev->data->port_id;
1682         q->queue_id = queue_id;
1683         q->tail_ptr = (volatile uint32_t *)
1684                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1685         if (handle_txconf(q, conf))
1686                 return (-EINVAL);
1687
1688         /* allocate memory for the software ring */
1689         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1690                                         nb_desc * sizeof(struct rte_mbuf *),
1691                                         RTE_CACHE_LINE_SIZE, socket_id);
1692         if (q->sw_ring == NULL) {
1693                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1694                 rte_free(q);
1695                 return (-ENOMEM);
1696         }
1697
1698         /*
1699          * allocate memory for the hardware descriptor ring. A memzone large
1700          * enough to hold the maximum ring size is requested to allow for
1701          * resizing in later calls to the queue setup function.
1702          */
1703         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1704                                 dev->data->port_id, queue_id, socket_id,
1705                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1706         if (mz == NULL) {
1707                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1708                 rte_free(q->sw_ring);
1709                 rte_free(q);
1710                 return (-ENOMEM);
1711         }
1712         q->hw_ring = mz->addr;
1713 #ifdef RTE_LIBRTE_XEN_DOM0
1714         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1715 #else
1716         q->hw_ring_phys_addr = mz->phys_addr;
1717 #endif
1718
1719         /*
1720          * allocate memory for the RS bit tracker. Enough slots to hold the
1721          * descriptor index for each RS bit needing to be set are required.
1722          */
1723         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1724                                 ((nb_desc + 1) / q->rs_thresh) *
1725                                 sizeof(uint16_t),
1726                                 RTE_CACHE_LINE_SIZE, socket_id);
1727         if (q->rs_tracker.list == NULL) {
1728                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1729                 rte_free(q->sw_ring);
1730                 rte_free(q);
1731                 return (-ENOMEM);
1732         }
1733
1734         dev->data->tx_queues[queue_id] = q;
1735         return 0;
1736 }
1737
1738 static void
1739 fm10k_tx_queue_release(void *queue)
1740 {
1741         PMD_INIT_FUNC_TRACE();
1742
1743         tx_queue_free(queue);
1744 }
1745
1746 static int
1747 fm10k_reta_update(struct rte_eth_dev *dev,
1748                         struct rte_eth_rss_reta_entry64 *reta_conf,
1749                         uint16_t reta_size)
1750 {
1751         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1752         uint16_t i, j, idx, shift;
1753         uint8_t mask;
1754         uint32_t reta;
1755
1756         PMD_INIT_FUNC_TRACE();
1757
1758         if (reta_size > FM10K_MAX_RSS_INDICES) {
1759                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1760                         "(%d) doesn't match the number hardware can supported "
1761                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1762                 return -EINVAL;
1763         }
1764
1765         /*
1766          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1767          * 128-entries in 32 registers
1768          */
1769         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1770                 idx = i / RTE_RETA_GROUP_SIZE;
1771                 shift = i % RTE_RETA_GROUP_SIZE;
1772                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1773                                 BIT_MASK_PER_UINT32);
1774                 if (mask == 0)
1775                         continue;
1776
1777                 reta = 0;
1778                 if (mask != BIT_MASK_PER_UINT32)
1779                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1780
1781                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1782                         if (mask & (0x1 << j)) {
1783                                 if (mask != 0xF)
1784                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1785                                 reta |= reta_conf[idx].reta[shift + j] <<
1786                                                 (CHAR_BIT * j);
1787                         }
1788                 }
1789                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1790         }
1791
1792         return 0;
1793 }
1794
1795 static int
1796 fm10k_reta_query(struct rte_eth_dev *dev,
1797                         struct rte_eth_rss_reta_entry64 *reta_conf,
1798                         uint16_t reta_size)
1799 {
1800         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1801         uint16_t i, j, idx, shift;
1802         uint8_t mask;
1803         uint32_t reta;
1804
1805         PMD_INIT_FUNC_TRACE();
1806
1807         if (reta_size < FM10K_MAX_RSS_INDICES) {
1808                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1809                         "(%d) doesn't match the number hardware can supported "
1810                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1811                 return -EINVAL;
1812         }
1813
1814         /*
1815          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1816          * 128-entries in 32 registers
1817          */
1818         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1819                 idx = i / RTE_RETA_GROUP_SIZE;
1820                 shift = i % RTE_RETA_GROUP_SIZE;
1821                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1822                                 BIT_MASK_PER_UINT32);
1823                 if (mask == 0)
1824                         continue;
1825
1826                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1827                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1828                         if (mask & (0x1 << j))
1829                                 reta_conf[idx].reta[shift + j] = ((reta >>
1830                                         CHAR_BIT * j) & UINT8_MAX);
1831                 }
1832         }
1833
1834         return 0;
1835 }
1836
1837 static int
1838 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1839         struct rte_eth_rss_conf *rss_conf)
1840 {
1841         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1842         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1843         uint32_t mrqc;
1844         uint64_t hf = rss_conf->rss_hf;
1845         int i;
1846
1847         PMD_INIT_FUNC_TRACE();
1848
1849         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1850                 FM10K_RSSRK_ENTRIES_PER_REG)
1851                 return -EINVAL;
1852
1853         if (hf == 0)
1854                 return -EINVAL;
1855
1856         mrqc = 0;
1857         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1858         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1859         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1860         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1861         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1862         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1863         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1864         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1865         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1866
1867         /* If the mapping doesn't fit any supported, return */
1868         if (mrqc == 0)
1869                 return -EINVAL;
1870
1871         if (key != NULL)
1872                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1873                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1874
1875         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1876
1877         return 0;
1878 }
1879
1880 static int
1881 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1882         struct rte_eth_rss_conf *rss_conf)
1883 {
1884         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1885         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1886         uint32_t mrqc;
1887         uint64_t hf;
1888         int i;
1889
1890         PMD_INIT_FUNC_TRACE();
1891
1892         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1893                                 FM10K_RSSRK_ENTRIES_PER_REG)
1894                 return -EINVAL;
1895
1896         if (key != NULL)
1897                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1898                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1899
1900         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1901         hf = 0;
1902         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1903         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1904         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1905         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1906         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1907         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1908         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1909         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1910         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1911
1912         rss_conf->rss_hf = hf;
1913
1914         return 0;
1915 }
1916
1917 static void
1918 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1919 {
1920         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1921         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1922
1923         /* Bind all local non-queue interrupt to vector 0 */
1924         int_map |= 0;
1925
1926         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1927         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1928         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1929         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1930         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1931         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1932
1933         /* Enable misc causes */
1934         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1935                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1936                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1937                                 FM10K_EIMR_ENABLE(MAILBOX) |
1938                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1939                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1940                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1941                                 FM10K_EIMR_ENABLE(VFLR));
1942
1943         /* Enable ITR 0 */
1944         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1945                                         FM10K_ITR_MASK_CLEAR);
1946         FM10K_WRITE_FLUSH(hw);
1947 }
1948
1949 static void
1950 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1951 {
1952         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1953         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1954
1955         int_map |= 0;
1956
1957         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1958         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1959         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1960         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1961         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1962         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1963
1964         /* Disable misc causes */
1965         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1966                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1967                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1968                                 FM10K_EIMR_DISABLE(MAILBOX) |
1969                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1970                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1971                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1972                                 FM10K_EIMR_DISABLE(VFLR));
1973
1974         /* Disable ITR 0 */
1975         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1976         FM10K_WRITE_FLUSH(hw);
1977 }
1978
1979 static void
1980 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1981 {
1982         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1983         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1984
1985         /* Bind all local non-queue interrupt to vector 0 */
1986         int_map |= 0;
1987
1988         /* Only INT 0 available, other 15 are reserved. */
1989         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1990
1991         /* Enable ITR 0 */
1992         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1993                                         FM10K_ITR_MASK_CLEAR);
1994         FM10K_WRITE_FLUSH(hw);
1995 }
1996
1997 static void
1998 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1999 {
2000         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2001         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2002
2003         int_map |= 0;
2004
2005         /* Only INT 0 available, other 15 are reserved. */
2006         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2007
2008         /* Disable ITR 0 */
2009         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2010         FM10K_WRITE_FLUSH(hw);
2011 }
2012
2013 static int
2014 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2015 {
2016         struct fm10k_fault fault;
2017         int err;
2018         const char *estr = "Unknown error";
2019
2020         /* Process PCA fault */
2021         if (eicr & FM10K_EICR_PCA_FAULT) {
2022                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2023                 if (err)
2024                         goto error;
2025                 switch (fault.type) {
2026                 case PCA_NO_FAULT:
2027                         estr = "PCA_NO_FAULT"; break;
2028                 case PCA_UNMAPPED_ADDR:
2029                         estr = "PCA_UNMAPPED_ADDR"; break;
2030                 case PCA_BAD_QACCESS_PF:
2031                         estr = "PCA_BAD_QACCESS_PF"; break;
2032                 case PCA_BAD_QACCESS_VF:
2033                         estr = "PCA_BAD_QACCESS_VF"; break;
2034                 case PCA_MALICIOUS_REQ:
2035                         estr = "PCA_MALICIOUS_REQ"; break;
2036                 case PCA_POISONED_TLP:
2037                         estr = "PCA_POISONED_TLP"; break;
2038                 case PCA_TLP_ABORT:
2039                         estr = "PCA_TLP_ABORT"; break;
2040                 default:
2041                         goto error;
2042                 }
2043                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2044                         estr, fault.func ? "VF" : "PF", fault.func,
2045                         fault.address, fault.specinfo);
2046         }
2047
2048         /* Process THI fault */
2049         if (eicr & FM10K_EICR_THI_FAULT) {
2050                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2051                 if (err)
2052                         goto error;
2053                 switch (fault.type) {
2054                 case THI_NO_FAULT:
2055                         estr = "THI_NO_FAULT"; break;
2056                 case THI_MAL_DIS_Q_FAULT:
2057                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2058                 default:
2059                         goto error;
2060                 }
2061                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2062                         estr, fault.func ? "VF" : "PF", fault.func,
2063                         fault.address, fault.specinfo);
2064         }
2065
2066         /* Process FUM fault */
2067         if (eicr & FM10K_EICR_FUM_FAULT) {
2068                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2069                 if (err)
2070                         goto error;
2071                 switch (fault.type) {
2072                 case FUM_NO_FAULT:
2073                         estr = "FUM_NO_FAULT"; break;
2074                 case FUM_UNMAPPED_ADDR:
2075                         estr = "FUM_UNMAPPED_ADDR"; break;
2076                 case FUM_POISONED_TLP:
2077                         estr = "FUM_POISONED_TLP"; break;
2078                 case FUM_BAD_VF_QACCESS:
2079                         estr = "FUM_BAD_VF_QACCESS"; break;
2080                 case FUM_ADD_DECODE_ERR:
2081                         estr = "FUM_ADD_DECODE_ERR"; break;
2082                 case FUM_RO_ERROR:
2083                         estr = "FUM_RO_ERROR"; break;
2084                 case FUM_QPRC_CRC_ERROR:
2085                         estr = "FUM_QPRC_CRC_ERROR"; break;
2086                 case FUM_CSR_TIMEOUT:
2087                         estr = "FUM_CSR_TIMEOUT"; break;
2088                 case FUM_INVALID_TYPE:
2089                         estr = "FUM_INVALID_TYPE"; break;
2090                 case FUM_INVALID_LENGTH:
2091                         estr = "FUM_INVALID_LENGTH"; break;
2092                 case FUM_INVALID_BE:
2093                         estr = "FUM_INVALID_BE"; break;
2094                 case FUM_INVALID_ALIGN:
2095                         estr = "FUM_INVALID_ALIGN"; break;
2096                 default:
2097                         goto error;
2098                 }
2099                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2100                         estr, fault.func ? "VF" : "PF", fault.func,
2101                         fault.address, fault.specinfo);
2102         }
2103
2104         return 0;
2105 error:
2106         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2107         return err;
2108 }
2109
2110 /**
2111  * PF interrupt handler triggered by NIC for handling specific interrupt.
2112  *
2113  * @param handle
2114  *  Pointer to interrupt handle.
2115  * @param param
2116  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2117  *
2118  * @return
2119  *  void
2120  */
2121 static void
2122 fm10k_dev_interrupt_handler_pf(
2123                         __rte_unused struct rte_intr_handle *handle,
2124                         void *param)
2125 {
2126         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2127         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2128         uint32_t cause, status;
2129
2130         if (hw->mac.type != fm10k_mac_pf)
2131                 return;
2132
2133         cause = FM10K_READ_REG(hw, FM10K_EICR);
2134
2135         /* Handle PCI fault cases */
2136         if (cause & FM10K_EICR_FAULT_MASK) {
2137                 PMD_INIT_LOG(ERR, "INT: find fault!");
2138                 fm10k_dev_handle_fault(hw, cause);
2139         }
2140
2141         /* Handle switch up/down */
2142         if (cause & FM10K_EICR_SWITCHNOTREADY)
2143                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2144
2145         if (cause & FM10K_EICR_SWITCHREADY)
2146                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2147
2148         /* Handle mailbox message */
2149         fm10k_mbx_lock(hw);
2150         hw->mbx.ops.process(hw, &hw->mbx);
2151         fm10k_mbx_unlock(hw);
2152
2153         /* Handle SRAM error */
2154         if (cause & FM10K_EICR_SRAMERROR) {
2155                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2156
2157                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2158                 /* Write to clear pending bits */
2159                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2160
2161                 /* Todo: print out error message after shared code  updates */
2162         }
2163
2164         /* Clear these 3 events if having any */
2165         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2166                  FM10K_EICR_SWITCHREADY;
2167         if (cause)
2168                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2169
2170         /* Re-enable interrupt from device side */
2171         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2172                                         FM10K_ITR_MASK_CLEAR);
2173         /* Re-enable interrupt from host side */
2174         rte_intr_enable(&(dev->pci_dev->intr_handle));
2175 }
2176
2177 /**
2178  * VF interrupt handler triggered by NIC for handling specific interrupt.
2179  *
2180  * @param handle
2181  *  Pointer to interrupt handle.
2182  * @param param
2183  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2184  *
2185  * @return
2186  *  void
2187  */
2188 static void
2189 fm10k_dev_interrupt_handler_vf(
2190                         __rte_unused struct rte_intr_handle *handle,
2191                         void *param)
2192 {
2193         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2194         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2195
2196         if (hw->mac.type != fm10k_mac_vf)
2197                 return;
2198
2199         /* Handle mailbox message if lock is acquired */
2200         fm10k_mbx_lock(hw);
2201         hw->mbx.ops.process(hw, &hw->mbx);
2202         fm10k_mbx_unlock(hw);
2203
2204         /* Re-enable interrupt from device side */
2205         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2206                                         FM10K_ITR_MASK_CLEAR);
2207         /* Re-enable interrupt from host side */
2208         rte_intr_enable(&(dev->pci_dev->intr_handle));
2209 }
2210
2211 /* Mailbox message handler in VF */
2212 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2213         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2214         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2215         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2216         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2217 };
2218
2219 /* Mailbox message handler in PF */
2220 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2221         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2222         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2223         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2224         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2225         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2226         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2227         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2228 };
2229
2230 static int
2231 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2232 {
2233         int err;
2234
2235         /* Initialize mailbox lock */
2236         fm10k_mbx_initlock(hw);
2237
2238         /* Replace default message handler with new ones */
2239         if (hw->mac.type == fm10k_mac_pf)
2240                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2241         else
2242                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2243
2244         if (err) {
2245                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2246                                 err);
2247                 return err;
2248         }
2249         /* Connect to SM for PF device or PF for VF device */
2250         return hw->mbx.ops.connect(hw, &hw->mbx);
2251 }
2252
2253 static void
2254 fm10k_close_mbx_service(struct fm10k_hw *hw)
2255 {
2256         /* Disconnect from SM for PF device or PF for VF device */
2257         hw->mbx.ops.disconnect(hw, &hw->mbx);
2258 }
2259
2260 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2261         .dev_configure          = fm10k_dev_configure,
2262         .dev_start              = fm10k_dev_start,
2263         .dev_stop               = fm10k_dev_stop,
2264         .dev_close              = fm10k_dev_close,
2265         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2266         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2267         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2268         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2269         .stats_get              = fm10k_stats_get,
2270         .stats_reset            = fm10k_stats_reset,
2271         .link_update            = fm10k_link_update,
2272         .dev_infos_get          = fm10k_dev_infos_get,
2273         .vlan_filter_set        = fm10k_vlan_filter_set,
2274         .vlan_offload_set       = fm10k_vlan_offload_set,
2275         .mac_addr_add           = fm10k_macaddr_add,
2276         .mac_addr_remove        = fm10k_macaddr_remove,
2277         .rx_queue_start         = fm10k_dev_rx_queue_start,
2278         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2279         .tx_queue_start         = fm10k_dev_tx_queue_start,
2280         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2281         .rx_queue_setup         = fm10k_rx_queue_setup,
2282         .rx_queue_release       = fm10k_rx_queue_release,
2283         .tx_queue_setup         = fm10k_tx_queue_setup,
2284         .tx_queue_release       = fm10k_tx_queue_release,
2285         .reta_update            = fm10k_reta_update,
2286         .reta_query             = fm10k_reta_query,
2287         .rss_hash_update        = fm10k_rss_hash_update,
2288         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2289 };
2290
2291 static void __attribute__((cold))
2292 fm10k_set_rx_function(struct rte_eth_dev *dev)
2293 {
2294         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2295         uint16_t i, rx_using_sse;
2296
2297         /* In order to allow Vector Rx there are a few configuration
2298          * conditions to be met.
2299          */
2300         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2301                 if (dev->data->scattered_rx)
2302                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2303                 else
2304                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2305         } else if (dev->data->scattered_rx)
2306                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2307
2308         rx_using_sse =
2309                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2310                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2311
2312         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2313                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2314
2315                 rxq->rx_using_sse = rx_using_sse;
2316         }
2317 }
2318
2319 static void
2320 fm10k_params_init(struct rte_eth_dev *dev)
2321 {
2322         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2323         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2324
2325         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2326          * there is no way to get link status without reading BAR4.  Until this
2327          * works, assume we have maximum bandwidth.
2328          * @todo - fix bus info
2329          */
2330         hw->bus_caps.speed = fm10k_bus_speed_8000;
2331         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2332         hw->bus_caps.payload = fm10k_bus_payload_512;
2333         hw->bus.speed = fm10k_bus_speed_8000;
2334         hw->bus.width = fm10k_bus_width_pcie_x8;
2335         hw->bus.payload = fm10k_bus_payload_256;
2336
2337         info->rx_vec_allowed = true;
2338 }
2339
2340 static int
2341 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2342 {
2343         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2344         int diag;
2345         struct fm10k_macvlan_filter_info *macvlan;
2346
2347         PMD_INIT_FUNC_TRACE();
2348
2349         dev->dev_ops = &fm10k_eth_dev_ops;
2350         dev->rx_pkt_burst = &fm10k_recv_pkts;
2351         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2352
2353         /* only initialize in the primary process */
2354         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2355                 return 0;
2356
2357         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2358         memset(macvlan, 0, sizeof(*macvlan));
2359         /* Vendor and Device ID need to be set before init of shared code */
2360         memset(hw, 0, sizeof(*hw));
2361         hw->device_id = dev->pci_dev->id.device_id;
2362         hw->vendor_id = dev->pci_dev->id.vendor_id;
2363         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2364         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2365         hw->revision_id = 0;
2366         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2367         if (hw->hw_addr == NULL) {
2368                 PMD_INIT_LOG(ERR, "Bad mem resource."
2369                         " Try to blacklist unused devices.");
2370                 return -EIO;
2371         }
2372
2373         /* Store fm10k_adapter pointer */
2374         hw->back = dev->data->dev_private;
2375
2376         /* Initialize the shared code */
2377         diag = fm10k_init_shared_code(hw);
2378         if (diag != FM10K_SUCCESS) {
2379                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2380                 return -EIO;
2381         }
2382
2383         /* Initialize parameters */
2384         fm10k_params_init(dev);
2385
2386         /* Initialize the hw */
2387         diag = fm10k_init_hw(hw);
2388         if (diag != FM10K_SUCCESS) {
2389                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2390                 return -EIO;
2391         }
2392
2393         /* Initialize MAC address(es) */
2394         dev->data->mac_addrs = rte_zmalloc("fm10k",
2395                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2396         if (dev->data->mac_addrs == NULL) {
2397                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2398                 return -ENOMEM;
2399         }
2400
2401         diag = fm10k_read_mac_addr(hw);
2402
2403         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2404                         &dev->data->mac_addrs[0]);
2405
2406         if (diag != FM10K_SUCCESS ||
2407                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2408
2409                 /* Generate a random addr */
2410                 eth_random_addr(hw->mac.addr);
2411                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2412                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2413                 &dev->data->mac_addrs[0]);
2414         }
2415
2416         /* Reset the hw statistics */
2417         fm10k_stats_reset(dev);
2418
2419         /* Reset the hw */
2420         diag = fm10k_reset_hw(hw);
2421         if (diag != FM10K_SUCCESS) {
2422                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2423                 return -EIO;
2424         }
2425
2426         /* Setup mailbox service */
2427         diag = fm10k_setup_mbx_service(hw);
2428         if (diag != FM10K_SUCCESS) {
2429                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2430                 return -EIO;
2431         }
2432
2433         /*PF/VF has different interrupt handling mechanism */
2434         if (hw->mac.type == fm10k_mac_pf) {
2435                 /* register callback func to eal lib */
2436                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2437                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2438
2439                 /* enable MISC interrupt */
2440                 fm10k_dev_enable_intr_pf(dev);
2441         } else { /* VF */
2442                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2443                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2444
2445                 fm10k_dev_enable_intr_vf(dev);
2446         }
2447
2448         /* Enable uio intr after callback registered */
2449         rte_intr_enable(&(dev->pci_dev->intr_handle));
2450
2451         hw->mac.ops.update_int_moderator(hw);
2452
2453         /* Make sure Switch Manager is ready before going forward. */
2454         if (hw->mac.type == fm10k_mac_pf) {
2455                 int switch_ready = 0;
2456                 int i;
2457
2458                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2459                         fm10k_mbx_lock(hw);
2460                         hw->mac.ops.get_host_state(hw, &switch_ready);
2461                         fm10k_mbx_unlock(hw);
2462                         if (switch_ready)
2463                                 break;
2464                         /* Delay some time to acquire async LPORT_MAP info. */
2465                         rte_delay_us(WAIT_SWITCH_MSG_US);
2466                 }
2467
2468                 if (switch_ready == 0) {
2469                         PMD_INIT_LOG(ERR, "switch is not ready");
2470                         return -1;
2471                 }
2472         }
2473
2474         /*
2475          * Below function will trigger operations on mailbox, acquire lock to
2476          * avoid race condition from interrupt handler. Operations on mailbox
2477          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2478          * will handle and generate an interrupt to our side. Then,  FIFO in
2479          * mailbox will be touched.
2480          */
2481         fm10k_mbx_lock(hw);
2482         /* Enable port first */
2483         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2484
2485         /* Set unicast mode by default. App can change to other mode in other
2486          * API func.
2487          */
2488         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2489                                         FM10K_XCAST_MODE_NONE);
2490
2491         fm10k_mbx_unlock(hw);
2492
2493         /* Add default mac address */
2494         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2495                 MAIN_VSI_POOL_NUMBER);
2496
2497         return 0;
2498 }
2499
2500 static int
2501 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2502 {
2503         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2504
2505         PMD_INIT_FUNC_TRACE();
2506
2507         /* only uninitialize in the primary process */
2508         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2509                 return 0;
2510
2511         /* safe to close dev here */
2512         fm10k_dev_close(dev);
2513
2514         dev->dev_ops = NULL;
2515         dev->rx_pkt_burst = NULL;
2516         dev->tx_pkt_burst = NULL;
2517
2518         /* disable uio/vfio intr */
2519         rte_intr_disable(&(dev->pci_dev->intr_handle));
2520
2521         /*PF/VF has different interrupt handling mechanism */
2522         if (hw->mac.type == fm10k_mac_pf) {
2523                 /* disable interrupt */
2524                 fm10k_dev_disable_intr_pf(dev);
2525
2526                 /* unregister callback func to eal lib */
2527                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2528                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2529         } else {
2530                 /* disable interrupt */
2531                 fm10k_dev_disable_intr_vf(dev);
2532
2533                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2534                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2535         }
2536
2537         /* free mac memory */
2538         if (dev->data->mac_addrs) {
2539                 rte_free(dev->data->mac_addrs);
2540                 dev->data->mac_addrs = NULL;
2541         }
2542
2543         memset(hw, 0, sizeof(*hw));
2544
2545         return 0;
2546 }
2547
2548 /*
2549  * The set of PCI devices this driver supports. This driver will enable both PF
2550  * and SRIOV-VF devices.
2551  */
2552 static const struct rte_pci_id pci_id_fm10k_map[] = {
2553 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2554 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2555 #include "rte_pci_dev_ids.h"
2556         { .vendor_id = 0, /* sentinel */ },
2557 };
2558
2559 static struct eth_driver rte_pmd_fm10k = {
2560         .pci_drv = {
2561                 .name = "rte_pmd_fm10k",
2562                 .id_table = pci_id_fm10k_map,
2563                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2564         },
2565         .eth_dev_init = eth_fm10k_dev_init,
2566         .eth_dev_uninit = eth_fm10k_dev_uninit,
2567         .dev_private_size = sizeof(struct fm10k_adapter),
2568 };
2569
2570 /*
2571  * Driver initialization routine.
2572  * Invoked once at EAL init time.
2573  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2574  */
2575 static int
2576 rte_pmd_fm10k_init(__rte_unused const char *name,
2577         __rte_unused const char *params)
2578 {
2579         PMD_INIT_FUNC_TRACE();
2580         rte_eth_driver_register(&rte_pmd_fm10k);
2581         return 0;
2582 }
2583
2584 static struct rte_driver rte_fm10k_driver = {
2585         .type = PMD_PDEV,
2586         .init = rte_pmd_fm10k_init,
2587 };
2588
2589 PMD_REGISTER_DRIVER(rte_fm10k_driver);