drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  64 static int
  65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  67         const u8 *mac, bool add, uint32_t pool);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
  71
  72 static void
  73 fm10k_mbx_initlock(struct fm10k_hw *hw)
  74 {
  75         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  76 }
  77
  78 static void
  79 fm10k_mbx_lock(struct fm10k_hw *hw)
  80 {
  81         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  82                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  83 }
  84
  85 static void
  86 fm10k_mbx_unlock(struct fm10k_hw *hw)
  87 {
  88         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  89 }
  90
  91 /*
  92  * reset queue to initial state, allocate software buffers used when starting
  93  * device.
  94  * return 0 on success
  95  * return -ENOMEM if buffers cannot be allocated
  96  * return -EINVAL if buffers do not satisfy alignment condition
  97  */
  98 static inline int
  99 rx_queue_reset(struct fm10k_rx_queue *q)
 100 {
 101         uint64_t dma_addr;
 102         int i, diag;
 103         PMD_INIT_FUNC_TRACE();
 104
 105         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 106         if (diag != 0)
 107                 return -ENOMEM;
 108
 109         for (i = 0; i < q->nb_desc; ++i) {
 110                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 111                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 112                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 113                                                 q->nb_desc);
 114                         return -EINVAL;
 115                 }
 116                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 117                 q->hw_ring[i].q.pkt_addr = dma_addr;
 118                 q->hw_ring[i].q.hdr_addr = dma_addr;
 119         }
 120
 121         q->next_dd = 0;
 122         q->next_alloc = 0;
 123         q->next_trigger = q->alloc_thresh - 1;
 124         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 125         q->rxrearm_start = 0;
 126         q->rxrearm_nb = 0;
 127
 128         return 0;
 129 }
 130
 131 /*
 132  * clean queue, descriptor rings, free software buffers used when stopping
 133  * device.
 134  */
 135 static inline void
 136 rx_queue_clean(struct fm10k_rx_queue *q)
 137 {
 138         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 139         uint32_t i;
 140         PMD_INIT_FUNC_TRACE();
 141
 142         /* zero descriptor rings */
 143         for (i = 0; i < q->nb_desc; ++i)
 144                 q->hw_ring[i] = zero;
 145
 146         /* vPMD driver has a different way of releasing mbufs. */
 147         if (q->rx_using_sse) {
 148                 fm10k_rx_queue_release_mbufs_vec(q);
 149                 return;
 150         }
 151
 152         /* free software buffers */
 153         for (i = 0; i < q->nb_desc; ++i) {
 154                 if (q->sw_ring[i]) {
 155                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 156                         q->sw_ring[i] = NULL;
 157                 }
 158         }
 159 }
 160
 161 /*
 162  * free all queue memory used when releasing the queue (i.e. configure)
 163  */
 164 static inline void
 165 rx_queue_free(struct fm10k_rx_queue *q)
 166 {
 167         PMD_INIT_FUNC_TRACE();
 168         if (q) {
 169                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 170                 rx_queue_clean(q);
 171                 if (q->sw_ring) {
 172                         rte_free(q->sw_ring);
 173                         q->sw_ring = NULL;
 174                 }
 175                 rte_free(q);
 176                 q = NULL;
 177         }
 178 }
 179
 180 /*
 181  * disable RX queue, wait unitl HW finished necessary flush operation
 182  */
 183 static inline int
 184 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 185 {
 186         uint32_t reg, i;
 187
 188         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 189         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 190                         reg & ~FM10K_RXQCTL_ENABLE);
 191
 192         /* Wait 100us at most */
 193         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 194                 rte_delay_us(1);
 195                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 196                 if (!(reg & FM10K_RXQCTL_ENABLE))
 197                         break;
 198         }
 199
 200         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 201                 return -1;
 202
 203         return 0;
 204 }
 205
 206 /*
 207  * reset queue to initial state, allocate software buffers used when starting
 208  * device
 209  */
 210 static inline void
 211 tx_queue_reset(struct fm10k_tx_queue *q)
 212 {
 213         PMD_INIT_FUNC_TRACE();
 214         q->last_free = 0;
 215         q->next_free = 0;
 216         q->nb_used = 0;
 217         q->nb_free = q->nb_desc - 1;
 218         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 219         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 220 }
 221
 222 /*
 223  * clean queue, descriptor rings, free software buffers used when stopping
 224  * device
 225  */
 226 static inline void
 227 tx_queue_clean(struct fm10k_tx_queue *q)
 228 {
 229         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 230         uint32_t i;
 231         PMD_INIT_FUNC_TRACE();
 232
 233         /* zero descriptor rings */
 234         for (i = 0; i < q->nb_desc; ++i)
 235                 q->hw_ring[i] = zero;
 236
 237         /* free software buffers */
 238         for (i = 0; i < q->nb_desc; ++i) {
 239                 if (q->sw_ring[i]) {
 240                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 241                         q->sw_ring[i] = NULL;
 242                 }
 243         }
 244 }
 245
 246 /*
 247  * free all queue memory used when releasing the queue (i.e. configure)
 248  */
 249 static inline void
 250 tx_queue_free(struct fm10k_tx_queue *q)
 251 {
 252         PMD_INIT_FUNC_TRACE();
 253         if (q) {
 254                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 255                 tx_queue_clean(q);
 256                 if (q->rs_tracker.list) {
 257                         rte_free(q->rs_tracker.list);
 258                         q->rs_tracker.list = NULL;
 259                 }
 260                 if (q->sw_ring) {
 261                         rte_free(q->sw_ring);
 262                         q->sw_ring = NULL;
 263                 }
 264                 rte_free(q);
 265                 q = NULL;
 266         }
 267 }
 268
 269 /*
 270  * disable TX queue, wait unitl HW finished necessary flush operation
 271  */
 272 static inline int
 273 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 274 {
 275         uint32_t reg, i;
 276
 277         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 278         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 279                         reg & ~FM10K_TXDCTL_ENABLE);
 280
 281         /* Wait 100us at most */
 282         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 283                 rte_delay_us(1);
 284                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 285                 if (!(reg & FM10K_TXDCTL_ENABLE))
 286                         break;
 287         }
 288
 289         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 290                 return -1;
 291
 292         return 0;
 293 }
 294
 295 static int
 296 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 297 {
 298         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 299         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 300         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 301         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 302
 303         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 304
 305         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 306                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 307                 return -EINVAL;
 308         }
 309
 310         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 311                 return 0;
 312
 313         if (hw->mac.type == fm10k_mac_vf) {
 314                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 315                 return -EINVAL;
 316         }
 317
 318         /* Check VMDQ queue pool number */
 319         if (vmdq_conf->nb_queue_pools >
 320                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 321                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 322                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 323                         vmdq_conf->nb_queue_pools);
 324                 return -EINVAL;
 325         }
 326
 327         return 0;
 328 }
 329
 330 static const struct fm10k_txq_ops def_txq_ops = {
 331         .release_mbufs = tx_queue_free,
 332         .reset = tx_queue_reset,
 333 };
 334
 335 static int
 336 fm10k_dev_configure(struct rte_eth_dev *dev)
 337 {
 338         int ret;
 339
 340         PMD_INIT_FUNC_TRACE();
 341
 342         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 343                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 344         /* multipe queue mode checking */
 345         ret  = fm10k_check_mq_mode(dev);
 346         if (ret != 0) {
 347                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 348                             ret);
 349                 return ret;
 350         }
 351
 352         return 0;
 353 }
 354
 355 /* fls = find last set bit = 32 minus the number of leading zeros */
 356 #ifndef fls
 357 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 358 #endif
 359
 360 static void
 361 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
 362 {
 363         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 364         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 365         uint32_t i;
 366
 367         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 368
 369         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
 370                 if (!vmdq_conf->pool_map[i].pools)
 371                         continue;
 372                 fm10k_mbx_lock(hw);
 373                 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
 374                 fm10k_mbx_unlock(hw);
 375         }
 376 }
 377
 378 static void
 379 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
 380 {
 381         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 382
 383         /* Add default mac address */
 384         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
 385                 MAIN_VSI_POOL_NUMBER);
 386 }
 387
 388 static void
 389 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
 390 {
 391         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 392         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 393         uint32_t mrqc, *key, i, reta, j;
 394         uint64_t hf;
 395
 396 #define RSS_KEY_SIZE 40
 397         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 398                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 399                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 400                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 401                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 402                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 403         };
 404
 405         if (dev->data->nb_rx_queues == 1 ||
 406             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 407             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 408                 return;
 409
 410         /* random key is rss_intel_key (default) or user provided (rss_key) */
 411         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 412                 key = (uint32_t *)rss_intel_key;
 413         else
 414                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 415
 416         /* Now fill our hash function seeds, 4 bytes at a time */
 417         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 418                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 419
 420         /*
 421          * Fill in redirection table
 422          * The byte-swap is needed because NIC registers are in
 423          * little-endian order.
 424          */
 425         reta = 0;
 426         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 427                 if (j == dev->data->nb_rx_queues)
 428                         j = 0;
 429                 reta = (reta << CHAR_BIT) | j;
 430                 if ((i & 3) == 3)
 431                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 432                                         rte_bswap32(reta));
 433         }
 434
 435         /*
 436          * Generate RSS hash based on packet types, TCP/UDP
 437          * port numbers and/or IPv4/v6 src and dst addresses
 438          */
 439         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 440         mrqc = 0;
 441         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 442         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 443         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 444         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 445         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 446         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 447         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 448         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 449         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 450
 451         if (mrqc == 0) {
 452                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 453                         "supported", hf);
 454                 return;
 455         }
 456
 457         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 458 }
 459
 460 static void
 461 fm10k_dev_logic_port_update(struct rte_eth_dev *dev,
 462         uint16_t nb_lport_old, uint16_t nb_lport_new)
 463 {
 464         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 465         uint32_t i;
 466
 467         fm10k_mbx_lock(hw);
 468         /* Disable previous logic ports */
 469         if (nb_lport_old)
 470                 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 471                         nb_lport_old, false);
 472         /* Enable new logic ports */
 473         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 474                 nb_lport_new, true);
 475         fm10k_mbx_unlock(hw);
 476
 477         for (i = 0; i < nb_lport_new; i++) {
 478                 /* Set unicast mode by default. App can change
 479                  * to other mode in other API func.
 480                  */
 481                 fm10k_mbx_lock(hw);
 482                 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
 483                         FM10K_XCAST_MODE_NONE);
 484                 fm10k_mbx_unlock(hw);
 485         }
 486 }
 487
 488 static void
 489 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 490 {
 491         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 492         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 493         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 494         struct fm10k_macvlan_filter_info *macvlan;
 495         uint16_t nb_queue_pools = 0; /* pool number in configuration */
 496         uint16_t nb_lport_new, nb_lport_old;
 497
 498         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 499         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 500
 501         fm10k_dev_rss_configure(dev);
 502
 503         /* only PF supports VMDQ */
 504         if (hw->mac.type != fm10k_mac_pf)
 505                 return;
 506
 507         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
 508                 nb_queue_pools = vmdq_conf->nb_queue_pools;
 509
 510         /* no pool number change, no need to update logic port and VLAN/MAC */
 511         if (macvlan->nb_queue_pools == nb_queue_pools)
 512                 return;
 513
 514         nb_lport_old = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 515         nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
 516         fm10k_dev_logic_port_update(dev, nb_lport_old, nb_lport_new);
 517
 518         /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
 519         memset(dev->data->mac_addrs, 0,
 520                 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
 521         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
 522                 &dev->data->mac_addrs[0]);
 523         memset(macvlan, 0, sizeof(*macvlan));
 524         macvlan->nb_queue_pools = nb_queue_pools;
 525
 526         if (nb_queue_pools)
 527                 fm10k_dev_vmdq_rx_configure(dev);
 528         else
 529                 fm10k_dev_pf_main_vsi_reset(dev);
 530 }
 531
 532 static int
 533 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 534 {
 535         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 536         int i, ret;
 537         struct fm10k_tx_queue *txq;
 538         uint64_t base_addr;
 539         uint32_t size;
 540
 541         /* Disable TXINT to avoid possible interrupt */
 542         for (i = 0; i < hw->mac.max_queues; i++)
 543                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 544                                 3 << FM10K_TXINT_TIMER_SHIFT);
 545
 546         /* Setup TX queue */
 547         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 548                 txq = dev->data->tx_queues[i];
 549                 base_addr = txq->hw_ring_phys_addr;
 550                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 551
 552                 /* disable queue to avoid issues while updating state */
 553                 ret = tx_queue_disable(hw, i);
 554                 if (ret) {
 555                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 556                         return -1;
 557                 }
 558
 559                 /* set location and size for descriptor ring */
 560                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 561                                 base_addr & UINT64_LOWER_32BITS_MASK);
 562                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 563                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 564                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 565         }
 566         return 0;
 567 }
 568
 569 static int
 570 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 571 {
 572         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 573         int i, ret;
 574         struct fm10k_rx_queue *rxq;
 575         uint64_t base_addr;
 576         uint32_t size;
 577         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 578         uint16_t buf_size;
 579
 580         /* Disable RXINT to avoid possible interrupt */
 581         for (i = 0; i < hw->mac.max_queues; i++)
 582                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 583                                 3 << FM10K_RXINT_TIMER_SHIFT);
 584
 585         /* Setup RX queues */
 586         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 587                 rxq = dev->data->rx_queues[i];
 588                 base_addr = rxq->hw_ring_phys_addr;
 589                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 590
 591                 /* disable queue to avoid issues while updating state */
 592                 ret = rx_queue_disable(hw, i);
 593                 if (ret) {
 594                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 595                         return -1;
 596                 }
 597
 598                 /* Setup the Base and Length of the Rx Descriptor Ring */
 599                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 600                                 base_addr & UINT64_LOWER_32BITS_MASK);
 601                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 602                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 603                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 604
 605                 /* Configure the Rx buffer size for one buff without split */
 606                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 607                         RTE_PKTMBUF_HEADROOM);
 608                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 609                  * reserved for this purpose, and the worst case could be 511B.
 610                  * But SRR reg assumes all buffers have the same size. In order
 611                  * to fill the gap, we'll have to consider the worst case and
 612                  * assume 512B is reserved. If we don't do so, it's possible
 613                  * for HW to overwrite data to next mbuf.
 614                  */
 615                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 616
 617                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 618                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 619
 620                 /* It adds dual VLAN length for supporting dual VLAN */
 621                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 622                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 623                         dev->data->dev_conf.rxmode.enable_scatter) {
 624                         uint32_t reg;
 625                         dev->data->scattered_rx = 1;
 626                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 627                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 628                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 629                 }
 630
 631                 /* Enable drop on empty, it's RO for VF */
 632                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 633                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 634
 635                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 636                 FM10K_WRITE_FLUSH(hw);
 637         }
 638
 639         /* Configure VMDQ/RSS if applicable */
 640         fm10k_dev_mq_rx_configure(dev);
 641
 642         /* Decide the best RX function */
 643         fm10k_set_rx_function(dev);
 644
 645         return 0;
 646 }
 647
 648 static int
 649 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 650 {
 651         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 652         int err = -1;
 653         uint32_t reg;
 654         struct fm10k_rx_queue *rxq;
 655
 656         PMD_INIT_FUNC_TRACE();
 657
 658         if (rx_queue_id < dev->data->nb_rx_queues) {
 659                 rxq = dev->data->rx_queues[rx_queue_id];
 660                 err = rx_queue_reset(rxq);
 661                 if (err == -ENOMEM) {
 662                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 663                         return err;
 664                 } else if (err == -EINVAL) {
 665                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 666                                 " %d", err);
 667                         return err;
 668                 }
 669
 670                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 671                  * Note: this must be done AFTER the queue is enabled on real
 672                  * hardware, but BEFORE the queue is enabled when using the
 673                  * emulation platform. Do it in both places for now and remove
 674                  * this comment and the following two register writes when the
 675                  * emulation platform is no longer being used.
 676                  */
 677                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 678                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 679
 680                 /* Set PF ownership flag for PF devices */
 681                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 682                 if (hw->mac.type == fm10k_mac_pf)
 683                         reg |= FM10K_RXQCTL_PF;
 684                 reg |= FM10K_RXQCTL_ENABLE;
 685                 /* enable RX queue */
 686                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 687                 FM10K_WRITE_FLUSH(hw);
 688
 689                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 690                  * Note: this must be done AFTER the queue is enabled
 691                  */
 692                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 693                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 694         }
 695
 696         return err;
 697 }
 698
 699 static int
 700 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 701 {
 702         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 703
 704         PMD_INIT_FUNC_TRACE();
 705
 706         if (rx_queue_id < dev->data->nb_rx_queues) {
 707                 /* Disable RX queue */
 708                 rx_queue_disable(hw, rx_queue_id);
 709
 710                 /* Free mbuf and clean HW ring */
 711                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 712         }
 713
 714         return 0;
 715 }
 716
 717 static int
 718 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 719 {
 720         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 721         /** @todo - this should be defined in the shared code */
 722 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 723         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 724         int err = 0;
 725
 726         PMD_INIT_FUNC_TRACE();
 727
 728         if (tx_queue_id < dev->data->nb_tx_queues) {
 729                 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
 730
 731                 q->ops->reset(q);
 732
 733                 /* reset head and tail pointers */
 734                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 735                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 736
 737                 /* enable TX queue */
 738                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 739                                         FM10K_TXDCTL_ENABLE | txdctl);
 740                 FM10K_WRITE_FLUSH(hw);
 741         } else
 742                 err = -1;
 743
 744         return err;
 745 }
 746
 747 static int
 748 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 749 {
 750         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 751
 752         PMD_INIT_FUNC_TRACE();
 753
 754         if (tx_queue_id < dev->data->nb_tx_queues) {
 755                 tx_queue_disable(hw, tx_queue_id);
 756                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 757         }
 758
 759         return 0;
 760 }
 761
 762 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 763 {
 764         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 765                 != FM10K_DGLORTMAP_NONE);
 766 }
 767
 768 static void
 769 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 770 {
 771         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 772         int status;
 773
 774         PMD_INIT_FUNC_TRACE();
 775
 776         /* Return if it didn't acquire valid glort range */
 777         if (!fm10k_glort_valid(hw))
 778                 return;
 779
 780         fm10k_mbx_lock(hw);
 781         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 782                                 FM10K_XCAST_MODE_PROMISC);
 783         fm10k_mbx_unlock(hw);
 784
 785         if (status != FM10K_SUCCESS)
 786                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 787 }
 788
 789 static void
 790 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 791 {
 792         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 793         uint8_t mode;
 794         int status;
 795
 796         PMD_INIT_FUNC_TRACE();
 797
 798         /* Return if it didn't acquire valid glort range */
 799         if (!fm10k_glort_valid(hw))
 800                 return;
 801
 802         if (dev->data->all_multicast == 1)
 803                 mode = FM10K_XCAST_MODE_ALLMULTI;
 804         else
 805                 mode = FM10K_XCAST_MODE_NONE;
 806
 807         fm10k_mbx_lock(hw);
 808         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 809                                 mode);
 810         fm10k_mbx_unlock(hw);
 811
 812         if (status != FM10K_SUCCESS)
 813                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 814 }
 815
 816 static void
 817 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 818 {
 819         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 820         int status;
 821
 822         PMD_INIT_FUNC_TRACE();
 823
 824         /* Return if it didn't acquire valid glort range */
 825         if (!fm10k_glort_valid(hw))
 826                 return;
 827
 828         /* If promiscuous mode is enabled, it doesn't make sense to enable
 829          * allmulticast and disable promiscuous since fm10k only can select
 830          * one of the modes.
 831          */
 832         if (dev->data->promiscuous) {
 833                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 834                         "needn't enable allmulticast");
 835                 return;
 836         }
 837
 838         fm10k_mbx_lock(hw);
 839         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 840                                 FM10K_XCAST_MODE_ALLMULTI);
 841         fm10k_mbx_unlock(hw);
 842
 843         if (status != FM10K_SUCCESS)
 844                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 845 }
 846
 847 static void
 848 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 849 {
 850         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 851         int status;
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         /* Return if it didn't acquire valid glort range */
 856         if (!fm10k_glort_valid(hw))
 857                 return;
 858
 859         if (dev->data->promiscuous) {
 860                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 861                         "since promisc mode is enabled");
 862                 return;
 863         }
 864
 865         fm10k_mbx_lock(hw);
 866         /* Change mode to unicast mode */
 867         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 868                                 FM10K_XCAST_MODE_NONE);
 869         fm10k_mbx_unlock(hw);
 870
 871         if (status != FM10K_SUCCESS)
 872                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 873 }
 874
 875 static void
 876 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
 877 {
 878         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 879         uint32_t dglortdec, pool_len, rss_len, i;
 880         uint16_t nb_queue_pools;
 881         struct fm10k_macvlan_filter_info *macvlan;
 882
 883         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 884         nb_queue_pools = macvlan->nb_queue_pools;
 885         pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
 886         rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
 887         dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
 888
 889         /* Establish only MAP 0 as valid */
 890         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 891
 892         /* Configure VMDQ/RSS DGlort Decoder */
 893         FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
 894
 895         /* Invalidate all other GLORT entries */
 896         for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 897                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 898                                 FM10K_DGLORTMAP_NONE);
 899 }
 900
 901 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 902 static int
 903 fm10k_dev_start(struct rte_eth_dev *dev)
 904 {
 905         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 906         int i, diag;
 907
 908         PMD_INIT_FUNC_TRACE();
 909
 910         /* stop, init, then start the hw */
 911         diag = fm10k_stop_hw(hw);
 912         if (diag != FM10K_SUCCESS) {
 913                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 914                 return -EIO;
 915         }
 916
 917         diag = fm10k_init_hw(hw);
 918         if (diag != FM10K_SUCCESS) {
 919                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 920                 return -EIO;
 921         }
 922
 923         diag = fm10k_start_hw(hw);
 924         if (diag != FM10K_SUCCESS) {
 925                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 926                 return -EIO;
 927         }
 928
 929         diag = fm10k_dev_tx_init(dev);
 930         if (diag) {
 931                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 932                 return diag;
 933         }
 934
 935         diag = fm10k_dev_rx_init(dev);
 936         if (diag) {
 937                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 938                 return diag;
 939         }
 940
 941         if (hw->mac.type == fm10k_mac_pf)
 942                 fm10k_dev_dglort_map_configure(dev);
 943
 944         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 945                 struct fm10k_rx_queue *rxq;
 946                 rxq = dev->data->rx_queues[i];
 947
 948                 if (rxq->rx_deferred_start)
 949                         continue;
 950                 diag = fm10k_dev_rx_queue_start(dev, i);
 951                 if (diag != 0) {
 952                         int j;
 953                         for (j = 0; j < i; ++j)
 954                                 rx_queue_clean(dev->data->rx_queues[j]);
 955                         return diag;
 956                 }
 957         }
 958
 959         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 960                 struct fm10k_tx_queue *txq;
 961                 txq = dev->data->tx_queues[i];
 962
 963                 if (txq->tx_deferred_start)
 964                         continue;
 965                 diag = fm10k_dev_tx_queue_start(dev, i);
 966                 if (diag != 0) {
 967                         int j;
 968                         for (j = 0; j < i; ++j)
 969                                 tx_queue_clean(dev->data->tx_queues[j]);
 970                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 971                                 rx_queue_clean(dev->data->rx_queues[j]);
 972                         return diag;
 973                 }
 974         }
 975
 976         /* Update default vlan when not in VMDQ mode */
 977         if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 978                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 979
 980         return 0;
 981 }
 982
 983 static void
 984 fm10k_dev_stop(struct rte_eth_dev *dev)
 985 {
 986         int i;
 987
 988         PMD_INIT_FUNC_TRACE();
 989
 990         if (dev->data->tx_queues)
 991                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 992                         fm10k_dev_tx_queue_stop(dev, i);
 993
 994         if (dev->data->rx_queues)
 995                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 996                         fm10k_dev_rx_queue_stop(dev, i);
 997 }
 998
 999 static void
1000 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1001 {
1002         int i;
1003
1004         PMD_INIT_FUNC_TRACE();
1005
1006         if (dev->data->tx_queues) {
1007                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1008                         struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1009
1010                         txq->ops->release_mbufs(txq);
1011                 }
1012         }
1013
1014         if (dev->data->rx_queues) {
1015                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1016                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
1017         }
1018 }
1019
1020 static void
1021 fm10k_dev_close(struct rte_eth_dev *dev)
1022 {
1023         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1024         uint16_t nb_lport;
1025         struct fm10k_macvlan_filter_info *macvlan;
1026
1027         PMD_INIT_FUNC_TRACE();
1028
1029         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1030         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
1031         fm10k_mbx_lock(hw);
1032         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1033                 nb_lport, false);
1034         fm10k_mbx_unlock(hw);
1035
1036         /* Stop mailbox service first */
1037         fm10k_close_mbx_service(hw);
1038         fm10k_dev_stop(dev);
1039         fm10k_dev_queue_release(dev);
1040         fm10k_stop_hw(hw);
1041 }
1042
1043 static int
1044 fm10k_link_update(struct rte_eth_dev *dev,
1045         __rte_unused int wait_to_complete)
1046 {
1047         PMD_INIT_FUNC_TRACE();
1048
1049         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
1050          * x8 PCIe interface. For now, we leave the speed undefined since there
1051          * is no 50Gbps Ethernet. */
1052         dev->data->dev_link.link_speed  = 0;
1053         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1054         dev->data->dev_link.link_status = 1;
1055
1056         return 0;
1057 }
1058
1059 static void
1060 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1061 {
1062         uint64_t ipackets, opackets, ibytes, obytes;
1063         struct fm10k_hw *hw =
1064                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1065         struct fm10k_hw_stats *hw_stats =
1066                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1067         int i;
1068
1069         PMD_INIT_FUNC_TRACE();
1070
1071         fm10k_update_hw_stats(hw, hw_stats);
1072
1073         ipackets = opackets = ibytes = obytes = 0;
1074         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1075                 (i < hw->mac.max_queues); ++i) {
1076                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1077                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1078                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
1079                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
1080                 ipackets += stats->q_ipackets[i];
1081                 opackets += stats->q_opackets[i];
1082                 ibytes   += stats->q_ibytes[i];
1083                 obytes   += stats->q_obytes[i];
1084         }
1085         stats->ipackets = ipackets;
1086         stats->opackets = opackets;
1087         stats->ibytes = ibytes;
1088         stats->obytes = obytes;
1089 }
1090
1091 static void
1092 fm10k_stats_reset(struct rte_eth_dev *dev)
1093 {
1094         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1095         struct fm10k_hw_stats *hw_stats =
1096                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1097
1098         PMD_INIT_FUNC_TRACE();
1099
1100         memset(hw_stats, 0, sizeof(*hw_stats));
1101         fm10k_rebind_hw_stats(hw, hw_stats);
1102 }
1103
1104 static void
1105 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1106         struct rte_eth_dev_info *dev_info)
1107 {
1108         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1109
1110         PMD_INIT_FUNC_TRACE();
1111
1112         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
1113         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
1114         dev_info->max_rx_queues      = hw->mac.max_queues;
1115         dev_info->max_tx_queues      = hw->mac.max_queues;
1116         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
1117         dev_info->max_hash_mac_addrs = 0;
1118         dev_info->max_vfs            = dev->pci_dev->max_vfs;
1119         dev_info->vmdq_pool_base     = 0;
1120         dev_info->vmdq_queue_base    = 0;
1121         dev_info->max_vmdq_pools     = ETH_32_POOLS;
1122         dev_info->vmdq_queue_num     = FM10K_MAX_QUEUES_PF;
1123         dev_info->rx_offload_capa =
1124                 DEV_RX_OFFLOAD_VLAN_STRIP |
1125                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1126                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1127                 DEV_RX_OFFLOAD_TCP_CKSUM;
1128         dev_info->tx_offload_capa =
1129                 DEV_TX_OFFLOAD_VLAN_INSERT |
1130                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1131                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1132                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1133                 DEV_TX_OFFLOAD_TCP_TSO;
1134
1135         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1136         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1137
1138         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1139                 .rx_thresh = {
1140                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1141                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1142                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1143                 },
1144                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1145                 .rx_drop_en = 0,
1146         };
1147
1148         dev_info->default_txconf = (struct rte_eth_txconf) {
1149                 .tx_thresh = {
1150                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1151                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1152                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1153                 },
1154                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1155                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1156                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1157                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1158         };
1159
1160         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1161                 .nb_max = FM10K_MAX_RX_DESC,
1162                 .nb_min = FM10K_MIN_RX_DESC,
1163                 .nb_align = FM10K_MULT_RX_DESC,
1164         };
1165
1166         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1167                 .nb_max = FM10K_MAX_TX_DESC,
1168                 .nb_min = FM10K_MIN_TX_DESC,
1169                 .nb_align = FM10K_MULT_TX_DESC,
1170         };
1171 }
1172
1173 static int
1174 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1175 {
1176         s32 result;
1177         uint16_t mac_num = 0;
1178         uint32_t vid_idx, vid_bit, mac_index;
1179         struct fm10k_hw *hw;
1180         struct fm10k_macvlan_filter_info *macvlan;
1181         struct rte_eth_dev_data *data = dev->data;
1182
1183         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1184         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1185
1186         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1187                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1188                 return (-EINVAL);
1189         }
1190
1191         if (vlan_id > ETH_VLAN_ID_MAX) {
1192                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1193                 return (-EINVAL);
1194         }
1195
1196         vid_idx = FM10K_VFTA_IDX(vlan_id);
1197         vid_bit = FM10K_VFTA_BIT(vlan_id);
1198         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1199         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1200                 return 0;
1201         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1202         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1203                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1204                         "in the VLAN filter table");
1205                 return (-EINVAL);
1206         }
1207
1208         fm10k_mbx_lock(hw);
1209         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1210         fm10k_mbx_unlock(hw);
1211         if (result != FM10K_SUCCESS) {
1212                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1213                 return (-EIO);
1214         }
1215
1216         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1217                         (result == FM10K_SUCCESS); mac_index++) {
1218                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1219                         continue;
1220                 if (mac_num > macvlan->mac_num - 1) {
1221                         PMD_INIT_LOG(ERR, "MAC address number "
1222                                         "not match");
1223                         break;
1224                 }
1225                 fm10k_mbx_lock(hw);
1226                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1227                         data->mac_addrs[mac_index].addr_bytes,
1228                         vlan_id, on, 0);
1229                 fm10k_mbx_unlock(hw);
1230                 mac_num++;
1231         }
1232         if (result != FM10K_SUCCESS) {
1233                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1234                 return (-EIO);
1235         }
1236
1237         if (on) {
1238                 macvlan->vlan_num++;
1239                 macvlan->vfta[vid_idx] |= vid_bit;
1240         } else {
1241                 macvlan->vlan_num--;
1242                 macvlan->vfta[vid_idx] &= ~vid_bit;
1243         }
1244         return 0;
1245 }
1246
1247 static void
1248 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1249 {
1250         if (mask & ETH_VLAN_STRIP_MASK) {
1251                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1252                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1253                                         "always on in fm10k");
1254         }
1255
1256         if (mask & ETH_VLAN_EXTEND_MASK) {
1257                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1258                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1259                                         "supported in fm10k");
1260         }
1261
1262         if (mask & ETH_VLAN_FILTER_MASK) {
1263                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1264                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1265         }
1266 }
1267
1268 /* Add/Remove a MAC address, and update filters to main VSI */
1269 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1270                 const u8 *mac, bool add, uint32_t pool)
1271 {
1272         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1273         struct fm10k_macvlan_filter_info *macvlan;
1274         uint32_t i, j, k;
1275
1276         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1277
1278         if (pool != MAIN_VSI_POOL_NUMBER) {
1279                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1280                         "mac to pool %u", pool);
1281                 return;
1282         }
1283         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1284                 if (!macvlan->vfta[j])
1285                         continue;
1286                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1287                         if (!(macvlan->vfta[j] & (1 << k)))
1288                                 continue;
1289                         if (i + 1 > macvlan->vlan_num) {
1290                                 PMD_INIT_LOG(ERR, "vlan number not match");
1291                                 return;
1292                         }
1293                         fm10k_mbx_lock(hw);
1294                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1295                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1296                         fm10k_mbx_unlock(hw);
1297                         i++;
1298                 }
1299         }
1300 }
1301
1302 /* Add/Remove a MAC address, and update filters to VMDQ */
1303 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1304                 const u8 *mac, bool add, uint32_t pool)
1305 {
1306         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1307         struct fm10k_macvlan_filter_info *macvlan;
1308         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1309         uint32_t i;
1310
1311         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1312         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1313
1314         if (pool > macvlan->nb_queue_pools) {
1315                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1316                         " Max pool is %u",
1317                         pool, macvlan->nb_queue_pools);
1318                 return;
1319         }
1320         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1321                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1322                         continue;
1323                 fm10k_mbx_lock(hw);
1324                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1325                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1326                 fm10k_mbx_unlock(hw);
1327         }
1328 }
1329
1330 /* Add/Remove a MAC address, and update filters */
1331 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1332                 const u8 *mac, bool add, uint32_t pool)
1333 {
1334         struct fm10k_macvlan_filter_info *macvlan;
1335
1336         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1337
1338         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1339                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1340         else
1341                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1342
1343         if (add)
1344                 macvlan->mac_num++;
1345         else
1346                 macvlan->mac_num--;
1347 }
1348
1349 /* Add a MAC address, and update filters */
1350 static void
1351 fm10k_macaddr_add(struct rte_eth_dev *dev,
1352                 struct ether_addr *mac_addr,
1353                 uint32_t index,
1354                 uint32_t pool)
1355 {
1356         struct fm10k_macvlan_filter_info *macvlan;
1357
1358         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1359         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1360         macvlan->mac_vmdq_id[index] = pool;
1361 }
1362
1363 /* Remove a MAC address, and update filters */
1364 static void
1365 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1366 {
1367         struct rte_eth_dev_data *data = dev->data;
1368         struct fm10k_macvlan_filter_info *macvlan;
1369
1370         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1371         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1372                         FALSE, macvlan->mac_vmdq_id[index]);
1373         macvlan->mac_vmdq_id[index] = 0;
1374 }
1375
1376 static inline int
1377 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1378 {
1379         if ((request < min) || (request > max) || ((request % mult) != 0))
1380                 return -1;
1381         else
1382                 return 0;
1383 }
1384
1385 /*
1386  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1387  * the physical address is required. If the memzone is already created, then
1388  * this function returns a pointer to the existing memzone.
1389  */
1390 static inline const struct rte_memzone *
1391 allocate_hw_ring(const char *driver_name, const char *ring_name,
1392         uint8_t port_id, uint16_t queue_id, int socket_id,
1393         uint32_t size, uint32_t align)
1394 {
1395         char name[RTE_MEMZONE_NAMESIZE];
1396         const struct rte_memzone *mz;
1397
1398         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1399                  driver_name, ring_name, port_id, queue_id, socket_id);
1400
1401         /* return the memzone if it already exists */
1402         mz = rte_memzone_lookup(name);
1403         if (mz)
1404                 return mz;
1405
1406 #ifdef RTE_LIBRTE_XEN_DOM0
1407         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1408                                            RTE_PGSIZE_2M);
1409 #else
1410         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1411 #endif
1412 }
1413
1414 static inline int
1415 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1416 {
1417         if ((request < min) || (request > max) || ((div % request) != 0))
1418                 return -1;
1419         else
1420                 return 0;
1421 }
1422
1423 static inline int
1424 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1425 {
1426         uint16_t rx_free_thresh;
1427
1428         if (conf->rx_free_thresh == 0)
1429                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1430         else
1431                 rx_free_thresh = conf->rx_free_thresh;
1432
1433         /* make sure the requested threshold satisfies the constraints */
1434         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1435                         FM10K_RX_FREE_THRESH_MAX(q),
1436                         FM10K_RX_FREE_THRESH_DIV(q),
1437                         rx_free_thresh)) {
1438                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1439                         "less than or equal to %u, "
1440                         "greater than or equal to %u, "
1441                         "and a divisor of %u",
1442                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1443                         FM10K_RX_FREE_THRESH_MIN(q),
1444                         FM10K_RX_FREE_THRESH_DIV(q));
1445                 return (-EINVAL);
1446         }
1447
1448         q->alloc_thresh = rx_free_thresh;
1449         q->drop_en = conf->rx_drop_en;
1450         q->rx_deferred_start = conf->rx_deferred_start;
1451
1452         return 0;
1453 }
1454
1455 /*
1456  * Hardware requires specific alignment for Rx packet buffers. At
1457  * least one of the following two conditions must be satisfied.
1458  *  1. Address is 512B aligned
1459  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1460  *
1461  * As such, the driver may need to adjust the DMA address within the
1462  * buffer by up to 512B.
1463  *
1464  * return 1 if the element size is valid, otherwise return 0.
1465  */
1466 static int
1467 mempool_element_size_valid(struct rte_mempool *mp)
1468 {
1469         uint32_t min_size;
1470
1471         /* elt_size includes mbuf header and headroom */
1472         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1473                         RTE_PKTMBUF_HEADROOM;
1474
1475         /* account for up to 512B of alignment */
1476         min_size -= FM10K_RX_DATABUF_ALIGN;
1477
1478         /* sanity check for overflow */
1479         if (min_size > mp->elt_size)
1480                 return 0;
1481
1482         /* size is valid */
1483         return 1;
1484 }
1485
1486 static int
1487 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1488         uint16_t nb_desc, unsigned int socket_id,
1489         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1490 {
1491         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1492         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1493         struct fm10k_rx_queue *q;
1494         const struct rte_memzone *mz;
1495
1496         PMD_INIT_FUNC_TRACE();
1497
1498         /* make sure the mempool element size can account for alignment. */
1499         if (!mempool_element_size_valid(mp)) {
1500                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1501                 return (-EINVAL);
1502         }
1503
1504         /* make sure a valid number of descriptors have been requested */
1505         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1506                                 FM10K_MULT_RX_DESC, nb_desc)) {
1507                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1508                         "less than or equal to %"PRIu32", "
1509                         "greater than or equal to %u, "
1510                         "and a multiple of %u",
1511                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1512                         FM10K_MULT_RX_DESC);
1513                 return (-EINVAL);
1514         }
1515
1516         /*
1517          * if this queue existed already, free the associated memory. The
1518          * queue cannot be reused in case we need to allocate memory on
1519          * different socket than was previously used.
1520          */
1521         if (dev->data->rx_queues[queue_id] != NULL) {
1522                 rx_queue_free(dev->data->rx_queues[queue_id]);
1523                 dev->data->rx_queues[queue_id] = NULL;
1524         }
1525
1526         /* allocate memory for the queue structure */
1527         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1528                                 socket_id);
1529         if (q == NULL) {
1530                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1531                 return (-ENOMEM);
1532         }
1533
1534         /* setup queue */
1535         q->mp = mp;
1536         q->nb_desc = nb_desc;
1537         q->port_id = dev->data->port_id;
1538         q->queue_id = queue_id;
1539         q->tail_ptr = (volatile uint32_t *)
1540                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1541         if (handle_rxconf(q, conf))
1542                 return (-EINVAL);
1543
1544         /* allocate memory for the software ring */
1545         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1546                                         nb_desc * sizeof(struct rte_mbuf *),
1547                                         RTE_CACHE_LINE_SIZE, socket_id);
1548         if (q->sw_ring == NULL) {
1549                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1550                 rte_free(q);
1551                 return (-ENOMEM);
1552         }
1553
1554         /*
1555          * allocate memory for the hardware descriptor ring. A memzone large
1556          * enough to hold the maximum ring size is requested to allow for
1557          * resizing in later calls to the queue setup function.
1558          */
1559         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1560                                 dev->data->port_id, queue_id, socket_id,
1561                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1562         if (mz == NULL) {
1563                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1564                 rte_free(q->sw_ring);
1565                 rte_free(q);
1566                 return (-ENOMEM);
1567         }
1568         q->hw_ring = mz->addr;
1569 #ifdef RTE_LIBRTE_XEN_DOM0
1570         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1571 #else
1572         q->hw_ring_phys_addr = mz->phys_addr;
1573 #endif
1574
1575         /* Check if number of descs satisfied Vector requirement */
1576         if (!rte_is_power_of_2(nb_desc)) {
1577                 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1578                                     "preconditions - canceling the feature for "
1579                                     "the whole port[%d]",
1580                              q->queue_id, q->port_id);
1581                 dev_info->rx_vec_allowed = false;
1582         } else
1583                 fm10k_rxq_vec_setup(q);
1584
1585         dev->data->rx_queues[queue_id] = q;
1586         return 0;
1587 }
1588
1589 static void
1590 fm10k_rx_queue_release(void *queue)
1591 {
1592         PMD_INIT_FUNC_TRACE();
1593
1594         rx_queue_free(queue);
1595 }
1596
1597 static inline int
1598 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1599 {
1600         uint16_t tx_free_thresh;
1601         uint16_t tx_rs_thresh;
1602
1603         /* constraint MACROs require that tx_free_thresh is configured
1604          * before tx_rs_thresh */
1605         if (conf->tx_free_thresh == 0)
1606                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1607         else
1608                 tx_free_thresh = conf->tx_free_thresh;
1609
1610         /* make sure the requested threshold satisfies the constraints */
1611         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1612                         FM10K_TX_FREE_THRESH_MAX(q),
1613                         FM10K_TX_FREE_THRESH_DIV(q),
1614                         tx_free_thresh)) {
1615                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1616                         "less than or equal to %u, "
1617                         "greater than or equal to %u, "
1618                         "and a divisor of %u",
1619                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1620                         FM10K_TX_FREE_THRESH_MIN(q),
1621                         FM10K_TX_FREE_THRESH_DIV(q));
1622                 return (-EINVAL);
1623         }
1624
1625         q->free_thresh = tx_free_thresh;
1626
1627         if (conf->tx_rs_thresh == 0)
1628                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1629         else
1630                 tx_rs_thresh = conf->tx_rs_thresh;
1631
1632         q->tx_deferred_start = conf->tx_deferred_start;
1633
1634         /* make sure the requested threshold satisfies the constraints */
1635         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1636                         FM10K_TX_RS_THRESH_MAX(q),
1637                         FM10K_TX_RS_THRESH_DIV(q),
1638                         tx_rs_thresh)) {
1639                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1640                         "less than or equal to %u, "
1641                         "greater than or equal to %u, "
1642                         "and a divisor of %u",
1643                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1644                         FM10K_TX_RS_THRESH_MIN(q),
1645                         FM10K_TX_RS_THRESH_DIV(q));
1646                 return (-EINVAL);
1647         }
1648
1649         q->rs_thresh = tx_rs_thresh;
1650
1651         return 0;
1652 }
1653
1654 static int
1655 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1656         uint16_t nb_desc, unsigned int socket_id,
1657         const struct rte_eth_txconf *conf)
1658 {
1659         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1660         struct fm10k_tx_queue *q;
1661         const struct rte_memzone *mz;
1662
1663         PMD_INIT_FUNC_TRACE();
1664
1665         /* make sure a valid number of descriptors have been requested */
1666         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1667                                 FM10K_MULT_TX_DESC, nb_desc)) {
1668                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1669                         "less than or equal to %"PRIu32", "
1670                         "greater than or equal to %u, "
1671                         "and a multiple of %u",
1672                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1673                         FM10K_MULT_TX_DESC);
1674                 return (-EINVAL);
1675         }
1676
1677         /*
1678          * if this queue existed already, free the associated memory. The
1679          * queue cannot be reused in case we need to allocate memory on
1680          * different socket than was previously used.
1681          */
1682         if (dev->data->tx_queues[queue_id] != NULL) {
1683                 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1684
1685                 txq->ops->release_mbufs(txq);
1686                 dev->data->tx_queues[queue_id] = NULL;
1687         }
1688
1689         /* allocate memory for the queue structure */
1690         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1691                                 socket_id);
1692         if (q == NULL) {
1693                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1694                 return (-ENOMEM);
1695         }
1696
1697         /* setup queue */
1698         q->nb_desc = nb_desc;
1699         q->port_id = dev->data->port_id;
1700         q->queue_id = queue_id;
1701         q->ops = &def_txq_ops;
1702         q->tail_ptr = (volatile uint32_t *)
1703                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1704         if (handle_txconf(q, conf))
1705                 return (-EINVAL);
1706
1707         /* allocate memory for the software ring */
1708         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1709                                         nb_desc * sizeof(struct rte_mbuf *),
1710                                         RTE_CACHE_LINE_SIZE, socket_id);
1711         if (q->sw_ring == NULL) {
1712                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1713                 rte_free(q);
1714                 return (-ENOMEM);
1715         }
1716
1717         /*
1718          * allocate memory for the hardware descriptor ring. A memzone large
1719          * enough to hold the maximum ring size is requested to allow for
1720          * resizing in later calls to the queue setup function.
1721          */
1722         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1723                                 dev->data->port_id, queue_id, socket_id,
1724                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1725         if (mz == NULL) {
1726                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1727                 rte_free(q->sw_ring);
1728                 rte_free(q);
1729                 return (-ENOMEM);
1730         }
1731         q->hw_ring = mz->addr;
1732 #ifdef RTE_LIBRTE_XEN_DOM0
1733         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1734 #else
1735         q->hw_ring_phys_addr = mz->phys_addr;
1736 #endif
1737
1738         /*
1739          * allocate memory for the RS bit tracker. Enough slots to hold the
1740          * descriptor index for each RS bit needing to be set are required.
1741          */
1742         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1743                                 ((nb_desc + 1) / q->rs_thresh) *
1744                                 sizeof(uint16_t),
1745                                 RTE_CACHE_LINE_SIZE, socket_id);
1746         if (q->rs_tracker.list == NULL) {
1747                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1748                 rte_free(q->sw_ring);
1749                 rte_free(q);
1750                 return (-ENOMEM);
1751         }
1752
1753         dev->data->tx_queues[queue_id] = q;
1754         return 0;
1755 }
1756
1757 static void
1758 fm10k_tx_queue_release(void *queue)
1759 {
1760         struct fm10k_tx_queue *q = queue;
1761         PMD_INIT_FUNC_TRACE();
1762
1763         q->ops->release_mbufs(q);
1764 }
1765
1766 static int
1767 fm10k_reta_update(struct rte_eth_dev *dev,
1768                         struct rte_eth_rss_reta_entry64 *reta_conf,
1769                         uint16_t reta_size)
1770 {
1771         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1772         uint16_t i, j, idx, shift;
1773         uint8_t mask;
1774         uint32_t reta;
1775
1776         PMD_INIT_FUNC_TRACE();
1777
1778         if (reta_size > FM10K_MAX_RSS_INDICES) {
1779                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1780                         "(%d) doesn't match the number hardware can supported "
1781                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1782                 return -EINVAL;
1783         }
1784
1785         /*
1786          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1787          * 128-entries in 32 registers
1788          */
1789         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1790                 idx = i / RTE_RETA_GROUP_SIZE;
1791                 shift = i % RTE_RETA_GROUP_SIZE;
1792                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1793                                 BIT_MASK_PER_UINT32);
1794                 if (mask == 0)
1795                         continue;
1796
1797                 reta = 0;
1798                 if (mask != BIT_MASK_PER_UINT32)
1799                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1800
1801                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1802                         if (mask & (0x1 << j)) {
1803                                 if (mask != 0xF)
1804                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1805                                 reta |= reta_conf[idx].reta[shift + j] <<
1806                                                 (CHAR_BIT * j);
1807                         }
1808                 }
1809                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1810         }
1811
1812         return 0;
1813 }
1814
1815 static int
1816 fm10k_reta_query(struct rte_eth_dev *dev,
1817                         struct rte_eth_rss_reta_entry64 *reta_conf,
1818                         uint16_t reta_size)
1819 {
1820         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1821         uint16_t i, j, idx, shift;
1822         uint8_t mask;
1823         uint32_t reta;
1824
1825         PMD_INIT_FUNC_TRACE();
1826
1827         if (reta_size < FM10K_MAX_RSS_INDICES) {
1828                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1829                         "(%d) doesn't match the number hardware can supported "
1830                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1831                 return -EINVAL;
1832         }
1833
1834         /*
1835          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1836          * 128-entries in 32 registers
1837          */
1838         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1839                 idx = i / RTE_RETA_GROUP_SIZE;
1840                 shift = i % RTE_RETA_GROUP_SIZE;
1841                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1842                                 BIT_MASK_PER_UINT32);
1843                 if (mask == 0)
1844                         continue;
1845
1846                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1847                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1848                         if (mask & (0x1 << j))
1849                                 reta_conf[idx].reta[shift + j] = ((reta >>
1850                                         CHAR_BIT * j) & UINT8_MAX);
1851                 }
1852         }
1853
1854         return 0;
1855 }
1856
1857 static int
1858 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1859         struct rte_eth_rss_conf *rss_conf)
1860 {
1861         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1862         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1863         uint32_t mrqc;
1864         uint64_t hf = rss_conf->rss_hf;
1865         int i;
1866
1867         PMD_INIT_FUNC_TRACE();
1868
1869         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1870                 FM10K_RSSRK_ENTRIES_PER_REG)
1871                 return -EINVAL;
1872
1873         if (hf == 0)
1874                 return -EINVAL;
1875
1876         mrqc = 0;
1877         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1878         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1879         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1880         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1881         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1882         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1883         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1884         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1885         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1886
1887         /* If the mapping doesn't fit any supported, return */
1888         if (mrqc == 0)
1889                 return -EINVAL;
1890
1891         if (key != NULL)
1892                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1893                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1894
1895         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1896
1897         return 0;
1898 }
1899
1900 static int
1901 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1902         struct rte_eth_rss_conf *rss_conf)
1903 {
1904         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1905         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1906         uint32_t mrqc;
1907         uint64_t hf;
1908         int i;
1909
1910         PMD_INIT_FUNC_TRACE();
1911
1912         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1913                                 FM10K_RSSRK_ENTRIES_PER_REG)
1914                 return -EINVAL;
1915
1916         if (key != NULL)
1917                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1918                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1919
1920         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1921         hf = 0;
1922         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1923         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1924         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1925         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1926         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1927         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1928         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1929         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1930         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1931
1932         rss_conf->rss_hf = hf;
1933
1934         return 0;
1935 }
1936
1937 static void
1938 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1939 {
1940         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1941         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1942
1943         /* Bind all local non-queue interrupt to vector 0 */
1944         int_map |= 0;
1945
1946         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1947         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1948         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1949         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1950         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1951         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1952
1953         /* Enable misc causes */
1954         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1955                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1956                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1957                                 FM10K_EIMR_ENABLE(MAILBOX) |
1958                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1959                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1960                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1961                                 FM10K_EIMR_ENABLE(VFLR));
1962
1963         /* Enable ITR 0 */
1964         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1965                                         FM10K_ITR_MASK_CLEAR);
1966         FM10K_WRITE_FLUSH(hw);
1967 }
1968
1969 static void
1970 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1971 {
1972         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1973         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1974
1975         int_map |= 0;
1976
1977         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1978         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1979         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1980         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1981         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1982         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1983
1984         /* Disable misc causes */
1985         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1986                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1987                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1988                                 FM10K_EIMR_DISABLE(MAILBOX) |
1989                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1990                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1991                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1992                                 FM10K_EIMR_DISABLE(VFLR));
1993
1994         /* Disable ITR 0 */
1995         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1996         FM10K_WRITE_FLUSH(hw);
1997 }
1998
1999 static void
2000 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2001 {
2002         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2003         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2004
2005         /* Bind all local non-queue interrupt to vector 0 */
2006         int_map |= 0;
2007
2008         /* Only INT 0 available, other 15 are reserved. */
2009         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2010
2011         /* Enable ITR 0 */
2012         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2013                                         FM10K_ITR_MASK_CLEAR);
2014         FM10K_WRITE_FLUSH(hw);
2015 }
2016
2017 static void
2018 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2019 {
2020         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2021         uint32_t int_map = FM10K_INT_MAP_DISABLE;
2022
2023         int_map |= 0;
2024
2025         /* Only INT 0 available, other 15 are reserved. */
2026         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2027
2028         /* Disable ITR 0 */
2029         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2030         FM10K_WRITE_FLUSH(hw);
2031 }
2032
2033 static int
2034 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2035 {
2036         struct fm10k_fault fault;
2037         int err;
2038         const char *estr = "Unknown error";
2039
2040         /* Process PCA fault */
2041         if (eicr & FM10K_EICR_PCA_FAULT) {
2042                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2043                 if (err)
2044                         goto error;
2045                 switch (fault.type) {
2046                 case PCA_NO_FAULT:
2047                         estr = "PCA_NO_FAULT"; break;
2048                 case PCA_UNMAPPED_ADDR:
2049                         estr = "PCA_UNMAPPED_ADDR"; break;
2050                 case PCA_BAD_QACCESS_PF:
2051                         estr = "PCA_BAD_QACCESS_PF"; break;
2052                 case PCA_BAD_QACCESS_VF:
2053                         estr = "PCA_BAD_QACCESS_VF"; break;
2054                 case PCA_MALICIOUS_REQ:
2055                         estr = "PCA_MALICIOUS_REQ"; break;
2056                 case PCA_POISONED_TLP:
2057                         estr = "PCA_POISONED_TLP"; break;
2058                 case PCA_TLP_ABORT:
2059                         estr = "PCA_TLP_ABORT"; break;
2060                 default:
2061                         goto error;
2062                 }
2063                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2064                         estr, fault.func ? "VF" : "PF", fault.func,
2065                         fault.address, fault.specinfo);
2066         }
2067
2068         /* Process THI fault */
2069         if (eicr & FM10K_EICR_THI_FAULT) {
2070                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2071                 if (err)
2072                         goto error;
2073                 switch (fault.type) {
2074                 case THI_NO_FAULT:
2075                         estr = "THI_NO_FAULT"; break;
2076                 case THI_MAL_DIS_Q_FAULT:
2077                         estr = "THI_MAL_DIS_Q_FAULT"; break;
2078                 default:
2079                         goto error;
2080                 }
2081                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2082                         estr, fault.func ? "VF" : "PF", fault.func,
2083                         fault.address, fault.specinfo);
2084         }
2085
2086         /* Process FUM fault */
2087         if (eicr & FM10K_EICR_FUM_FAULT) {
2088                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2089                 if (err)
2090                         goto error;
2091                 switch (fault.type) {
2092                 case FUM_NO_FAULT:
2093                         estr = "FUM_NO_FAULT"; break;
2094                 case FUM_UNMAPPED_ADDR:
2095                         estr = "FUM_UNMAPPED_ADDR"; break;
2096                 case FUM_POISONED_TLP:
2097                         estr = "FUM_POISONED_TLP"; break;
2098                 case FUM_BAD_VF_QACCESS:
2099                         estr = "FUM_BAD_VF_QACCESS"; break;
2100                 case FUM_ADD_DECODE_ERR:
2101                         estr = "FUM_ADD_DECODE_ERR"; break;
2102                 case FUM_RO_ERROR:
2103                         estr = "FUM_RO_ERROR"; break;
2104                 case FUM_QPRC_CRC_ERROR:
2105                         estr = "FUM_QPRC_CRC_ERROR"; break;
2106                 case FUM_CSR_TIMEOUT:
2107                         estr = "FUM_CSR_TIMEOUT"; break;
2108                 case FUM_INVALID_TYPE:
2109                         estr = "FUM_INVALID_TYPE"; break;
2110                 case FUM_INVALID_LENGTH:
2111                         estr = "FUM_INVALID_LENGTH"; break;
2112                 case FUM_INVALID_BE:
2113                         estr = "FUM_INVALID_BE"; break;
2114                 case FUM_INVALID_ALIGN:
2115                         estr = "FUM_INVALID_ALIGN"; break;
2116                 default:
2117                         goto error;
2118                 }
2119                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2120                         estr, fault.func ? "VF" : "PF", fault.func,
2121                         fault.address, fault.specinfo);
2122         }
2123
2124         return 0;
2125 error:
2126         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2127         return err;
2128 }
2129
2130 /**
2131  * PF interrupt handler triggered by NIC for handling specific interrupt.
2132  *
2133  * @param handle
2134  *  Pointer to interrupt handle.
2135  * @param param
2136  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2137  *
2138  * @return
2139  *  void
2140  */
2141 static void
2142 fm10k_dev_interrupt_handler_pf(
2143                         __rte_unused struct rte_intr_handle *handle,
2144                         void *param)
2145 {
2146         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2147         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2148         uint32_t cause, status;
2149
2150         if (hw->mac.type != fm10k_mac_pf)
2151                 return;
2152
2153         cause = FM10K_READ_REG(hw, FM10K_EICR);
2154
2155         /* Handle PCI fault cases */
2156         if (cause & FM10K_EICR_FAULT_MASK) {
2157                 PMD_INIT_LOG(ERR, "INT: find fault!");
2158                 fm10k_dev_handle_fault(hw, cause);
2159         }
2160
2161         /* Handle switch up/down */
2162         if (cause & FM10K_EICR_SWITCHNOTREADY)
2163                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2164
2165         if (cause & FM10K_EICR_SWITCHREADY)
2166                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2167
2168         /* Handle mailbox message */
2169         fm10k_mbx_lock(hw);
2170         hw->mbx.ops.process(hw, &hw->mbx);
2171         fm10k_mbx_unlock(hw);
2172
2173         /* Handle SRAM error */
2174         if (cause & FM10K_EICR_SRAMERROR) {
2175                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2176
2177                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2178                 /* Write to clear pending bits */
2179                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2180
2181                 /* Todo: print out error message after shared code  updates */
2182         }
2183
2184         /* Clear these 3 events if having any */
2185         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2186                  FM10K_EICR_SWITCHREADY;
2187         if (cause)
2188                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2189
2190         /* Re-enable interrupt from device side */
2191         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2192                                         FM10K_ITR_MASK_CLEAR);
2193         /* Re-enable interrupt from host side */
2194         rte_intr_enable(&(dev->pci_dev->intr_handle));
2195 }
2196
2197 /**
2198  * VF interrupt handler triggered by NIC for handling specific interrupt.
2199  *
2200  * @param handle
2201  *  Pointer to interrupt handle.
2202  * @param param
2203  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2204  *
2205  * @return
2206  *  void
2207  */
2208 static void
2209 fm10k_dev_interrupt_handler_vf(
2210                         __rte_unused struct rte_intr_handle *handle,
2211                         void *param)
2212 {
2213         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2214         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2215
2216         if (hw->mac.type != fm10k_mac_vf)
2217                 return;
2218
2219         /* Handle mailbox message if lock is acquired */
2220         fm10k_mbx_lock(hw);
2221         hw->mbx.ops.process(hw, &hw->mbx);
2222         fm10k_mbx_unlock(hw);
2223
2224         /* Re-enable interrupt from device side */
2225         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2226                                         FM10K_ITR_MASK_CLEAR);
2227         /* Re-enable interrupt from host side */
2228         rte_intr_enable(&(dev->pci_dev->intr_handle));
2229 }
2230
2231 /* Mailbox message handler in VF */
2232 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2233         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2234         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2235         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2236         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2237 };
2238
2239 /* Mailbox message handler in PF */
2240 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2241         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2242         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2243         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2244         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2245         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2246         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2247         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2248 };
2249
2250 static int
2251 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2252 {
2253         int err;
2254
2255         /* Initialize mailbox lock */
2256         fm10k_mbx_initlock(hw);
2257
2258         /* Replace default message handler with new ones */
2259         if (hw->mac.type == fm10k_mac_pf)
2260                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2261         else
2262                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2263
2264         if (err) {
2265                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2266                                 err);
2267                 return err;
2268         }
2269         /* Connect to SM for PF device or PF for VF device */
2270         return hw->mbx.ops.connect(hw, &hw->mbx);
2271 }
2272
2273 static void
2274 fm10k_close_mbx_service(struct fm10k_hw *hw)
2275 {
2276         /* Disconnect from SM for PF device or PF for VF device */
2277         hw->mbx.ops.disconnect(hw, &hw->mbx);
2278 }
2279
2280 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2281         .dev_configure          = fm10k_dev_configure,
2282         .dev_start              = fm10k_dev_start,
2283         .dev_stop               = fm10k_dev_stop,
2284         .dev_close              = fm10k_dev_close,
2285         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2286         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2287         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2288         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2289         .stats_get              = fm10k_stats_get,
2290         .stats_reset            = fm10k_stats_reset,
2291         .link_update            = fm10k_link_update,
2292         .dev_infos_get          = fm10k_dev_infos_get,
2293         .vlan_filter_set        = fm10k_vlan_filter_set,
2294         .vlan_offload_set       = fm10k_vlan_offload_set,
2295         .mac_addr_add           = fm10k_macaddr_add,
2296         .mac_addr_remove        = fm10k_macaddr_remove,
2297         .rx_queue_start         = fm10k_dev_rx_queue_start,
2298         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2299         .tx_queue_start         = fm10k_dev_tx_queue_start,
2300         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2301         .rx_queue_setup         = fm10k_rx_queue_setup,
2302         .rx_queue_release       = fm10k_rx_queue_release,
2303         .tx_queue_setup         = fm10k_tx_queue_setup,
2304         .tx_queue_release       = fm10k_tx_queue_release,
2305         .reta_update            = fm10k_reta_update,
2306         .reta_query             = fm10k_reta_query,
2307         .rss_hash_update        = fm10k_rss_hash_update,
2308         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2309 };
2310
2311 static void __attribute__((cold))
2312 fm10k_set_rx_function(struct rte_eth_dev *dev)
2313 {
2314         struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2315         uint16_t i, rx_using_sse;
2316
2317         /* In order to allow Vector Rx there are a few configuration
2318          * conditions to be met.
2319          */
2320         if (!fm10k_rx_vec_condition_check(dev) && dev_info->rx_vec_allowed) {
2321                 if (dev->data->scattered_rx)
2322                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2323                 else
2324                         dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2325         } else if (dev->data->scattered_rx)
2326                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2327
2328         rx_using_sse =
2329                 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2330                 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2331
2332         for (i = 0; i < dev->data->nb_rx_queues; i++) {
2333                 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2334
2335                 rxq->rx_using_sse = rx_using_sse;
2336         }
2337 }
2338
2339 static void
2340 fm10k_params_init(struct rte_eth_dev *dev)
2341 {
2342         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2343         struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2344
2345         /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2346          * there is no way to get link status without reading BAR4.  Until this
2347          * works, assume we have maximum bandwidth.
2348          * @todo - fix bus info
2349          */
2350         hw->bus_caps.speed = fm10k_bus_speed_8000;
2351         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2352         hw->bus_caps.payload = fm10k_bus_payload_512;
2353         hw->bus.speed = fm10k_bus_speed_8000;
2354         hw->bus.width = fm10k_bus_width_pcie_x8;
2355         hw->bus.payload = fm10k_bus_payload_256;
2356
2357         info->rx_vec_allowed = true;
2358 }
2359
2360 static int
2361 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2362 {
2363         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2364         int diag;
2365         struct fm10k_macvlan_filter_info *macvlan;
2366
2367         PMD_INIT_FUNC_TRACE();
2368
2369         dev->dev_ops = &fm10k_eth_dev_ops;
2370         dev->rx_pkt_burst = &fm10k_recv_pkts;
2371         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2372
2373         /* only initialize in the primary process */
2374         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2375                 return 0;
2376
2377         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2378         memset(macvlan, 0, sizeof(*macvlan));
2379         /* Vendor and Device ID need to be set before init of shared code */
2380         memset(hw, 0, sizeof(*hw));
2381         hw->device_id = dev->pci_dev->id.device_id;
2382         hw->vendor_id = dev->pci_dev->id.vendor_id;
2383         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2384         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2385         hw->revision_id = 0;
2386         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2387         if (hw->hw_addr == NULL) {
2388                 PMD_INIT_LOG(ERR, "Bad mem resource."
2389                         " Try to blacklist unused devices.");
2390                 return -EIO;
2391         }
2392
2393         /* Store fm10k_adapter pointer */
2394         hw->back = dev->data->dev_private;
2395
2396         /* Initialize the shared code */
2397         diag = fm10k_init_shared_code(hw);
2398         if (diag != FM10K_SUCCESS) {
2399                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2400                 return -EIO;
2401         }
2402
2403         /* Initialize parameters */
2404         fm10k_params_init(dev);
2405
2406         /* Initialize the hw */
2407         diag = fm10k_init_hw(hw);
2408         if (diag != FM10K_SUCCESS) {
2409                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2410                 return -EIO;
2411         }
2412
2413         /* Initialize MAC address(es) */
2414         dev->data->mac_addrs = rte_zmalloc("fm10k",
2415                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2416         if (dev->data->mac_addrs == NULL) {
2417                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2418                 return -ENOMEM;
2419         }
2420
2421         diag = fm10k_read_mac_addr(hw);
2422
2423         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2424                         &dev->data->mac_addrs[0]);
2425
2426         if (diag != FM10K_SUCCESS ||
2427                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2428
2429                 /* Generate a random addr */
2430                 eth_random_addr(hw->mac.addr);
2431                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2432                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2433                 &dev->data->mac_addrs[0]);
2434         }
2435
2436         /* Reset the hw statistics */
2437         fm10k_stats_reset(dev);
2438
2439         /* Reset the hw */
2440         diag = fm10k_reset_hw(hw);
2441         if (diag != FM10K_SUCCESS) {
2442                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2443                 return -EIO;
2444         }
2445
2446         /* Setup mailbox service */
2447         diag = fm10k_setup_mbx_service(hw);
2448         if (diag != FM10K_SUCCESS) {
2449                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2450                 return -EIO;
2451         }
2452
2453         /*PF/VF has different interrupt handling mechanism */
2454         if (hw->mac.type == fm10k_mac_pf) {
2455                 /* register callback func to eal lib */
2456                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2457                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2458
2459                 /* enable MISC interrupt */
2460                 fm10k_dev_enable_intr_pf(dev);
2461         } else { /* VF */
2462                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2463                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2464
2465                 fm10k_dev_enable_intr_vf(dev);
2466         }
2467
2468         /* Enable uio intr after callback registered */
2469         rte_intr_enable(&(dev->pci_dev->intr_handle));
2470
2471         hw->mac.ops.update_int_moderator(hw);
2472
2473         /* Make sure Switch Manager is ready before going forward. */
2474         if (hw->mac.type == fm10k_mac_pf) {
2475                 int switch_ready = 0;
2476                 int i;
2477
2478                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2479                         fm10k_mbx_lock(hw);
2480                         hw->mac.ops.get_host_state(hw, &switch_ready);
2481                         fm10k_mbx_unlock(hw);
2482                         if (switch_ready)
2483                                 break;
2484                         /* Delay some time to acquire async LPORT_MAP info. */
2485                         rte_delay_us(WAIT_SWITCH_MSG_US);
2486                 }
2487
2488                 if (switch_ready == 0) {
2489                         PMD_INIT_LOG(ERR, "switch is not ready");
2490                         return -1;
2491                 }
2492         }
2493
2494         /*
2495          * Below function will trigger operations on mailbox, acquire lock to
2496          * avoid race condition from interrupt handler. Operations on mailbox
2497          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2498          * will handle and generate an interrupt to our side. Then,  FIFO in
2499          * mailbox will be touched.
2500          */
2501         fm10k_mbx_lock(hw);
2502         /* Enable port first */
2503         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2504
2505         /* Set unicast mode by default. App can change to other mode in other
2506          * API func.
2507          */
2508         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2509                                         FM10K_XCAST_MODE_NONE);
2510
2511         fm10k_mbx_unlock(hw);
2512
2513         /* Add default mac address */
2514         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2515                 MAIN_VSI_POOL_NUMBER);
2516
2517         return 0;
2518 }
2519
2520 static int
2521 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2522 {
2523         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2524
2525         PMD_INIT_FUNC_TRACE();
2526
2527         /* only uninitialize in the primary process */
2528         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2529                 return 0;
2530
2531         /* safe to close dev here */
2532         fm10k_dev_close(dev);
2533
2534         dev->dev_ops = NULL;
2535         dev->rx_pkt_burst = NULL;
2536         dev->tx_pkt_burst = NULL;
2537
2538         /* disable uio/vfio intr */
2539         rte_intr_disable(&(dev->pci_dev->intr_handle));
2540
2541         /*PF/VF has different interrupt handling mechanism */
2542         if (hw->mac.type == fm10k_mac_pf) {
2543                 /* disable interrupt */
2544                 fm10k_dev_disable_intr_pf(dev);
2545
2546                 /* unregister callback func to eal lib */
2547                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2548                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2549         } else {
2550                 /* disable interrupt */
2551                 fm10k_dev_disable_intr_vf(dev);
2552
2553                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2554                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2555         }
2556
2557         /* free mac memory */
2558         if (dev->data->mac_addrs) {
2559                 rte_free(dev->data->mac_addrs);
2560                 dev->data->mac_addrs = NULL;
2561         }
2562
2563         memset(hw, 0, sizeof(*hw));
2564
2565         return 0;
2566 }
2567
2568 /*
2569  * The set of PCI devices this driver supports. This driver will enable both PF
2570  * and SRIOV-VF devices.
2571  */
2572 static const struct rte_pci_id pci_id_fm10k_map[] = {
2573 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2574 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2575 #include "rte_pci_dev_ids.h"
2576         { .vendor_id = 0, /* sentinel */ },
2577 };
2578
2579 static struct eth_driver rte_pmd_fm10k = {
2580         .pci_drv = {
2581                 .name = "rte_pmd_fm10k",
2582                 .id_table = pci_id_fm10k_map,
2583                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2584         },
2585         .eth_dev_init = eth_fm10k_dev_init,
2586         .eth_dev_uninit = eth_fm10k_dev_uninit,
2587         .dev_private_size = sizeof(struct fm10k_adapter),
2588 };
2589
2590 /*
2591  * Driver initialization routine.
2592  * Invoked once at EAL init time.
2593  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2594  */
2595 static int
2596 rte_pmd_fm10k_init(__rte_unused const char *name,
2597         __rte_unused const char *params)
2598 {
2599         PMD_INIT_FUNC_TRACE();
2600         rte_eth_driver_register(&rte_pmd_fm10k);
2601         return 0;
2602 }
2603
2604 static struct rte_driver rte_fm10k_driver = {
2605         .type = PMD_PDEV,
2606         .init = rte_pmd_fm10k_init,
2607 };
2608
2609 PMD_REGISTER_DRIVER(rte_fm10k_driver);