drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 #define FM10K_RX_BUFF_ALIGN 512
  45 /* Default delay to acquire mailbox lock */
  46 #define FM10K_MBXLOCK_DELAY_US 20
  47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  48
  49 /* Number of chars per uint32 type */
  50 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  51 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  52
  53 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  54 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  55 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  56 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  57 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  58 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  59
  60 static void
  61 fm10k_mbx_initlock(struct fm10k_hw *hw)
  62 {
  63         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  64 }
  65
  66 static void
  67 fm10k_mbx_lock(struct fm10k_hw *hw)
  68 {
  69         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  70                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  71 }
  72
  73 static void
  74 fm10k_mbx_unlock(struct fm10k_hw *hw)
  75 {
  76         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  77 }
  78
  79 /*
  80  * reset queue to initial state, allocate software buffers used when starting
  81  * device.
  82  * return 0 on success
  83  * return -ENOMEM if buffers cannot be allocated
  84  * return -EINVAL if buffers do not satisfy alignment condition
  85  */
  86 static inline int
  87 rx_queue_reset(struct fm10k_rx_queue *q)
  88 {
  89         uint64_t dma_addr;
  90         int i, diag;
  91         PMD_INIT_FUNC_TRACE();
  92
  93         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
  94         if (diag != 0)
  95                 return -ENOMEM;
  96
  97         for (i = 0; i < q->nb_desc; ++i) {
  98                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
  99                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 100                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 101                                                 q->nb_desc);
 102                         return -EINVAL;
 103                 }
 104                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 105                 q->hw_ring[i].q.pkt_addr = dma_addr;
 106                 q->hw_ring[i].q.hdr_addr = dma_addr;
 107         }
 108
 109         q->next_dd = 0;
 110         q->next_alloc = 0;
 111         q->next_trigger = q->alloc_thresh - 1;
 112         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 113         return 0;
 114 }
 115
 116 /*
 117  * clean queue, descriptor rings, free software buffers used when stopping
 118  * device.
 119  */
 120 static inline void
 121 rx_queue_clean(struct fm10k_rx_queue *q)
 122 {
 123         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 124         uint32_t i;
 125         PMD_INIT_FUNC_TRACE();
 126
 127         /* zero descriptor rings */
 128         for (i = 0; i < q->nb_desc; ++i)
 129                 q->hw_ring[i] = zero;
 130
 131         /* free software buffers */
 132         for (i = 0; i < q->nb_desc; ++i) {
 133                 if (q->sw_ring[i]) {
 134                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 135                         q->sw_ring[i] = NULL;
 136                 }
 137         }
 138 }
 139
 140 /*
 141  * free all queue memory used when releasing the queue (i.e. configure)
 142  */
 143 static inline void
 144 rx_queue_free(struct fm10k_rx_queue *q)
 145 {
 146         PMD_INIT_FUNC_TRACE();
 147         if (q) {
 148                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 149                 rx_queue_clean(q);
 150                 if (q->sw_ring) {
 151                         rte_free(q->sw_ring);
 152                         q->sw_ring = NULL;
 153                 }
 154                 rte_free(q);
 155                 q = NULL;
 156         }
 157 }
 158
 159 /*
 160  * disable RX queue, wait unitl HW finished necessary flush operation
 161  */
 162 static inline int
 163 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 164 {
 165         uint32_t reg, i;
 166
 167         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 168         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 169                         reg & ~FM10K_RXQCTL_ENABLE);
 170
 171         /* Wait 100us at most */
 172         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 173                 rte_delay_us(1);
 174                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 175                 if (!(reg & FM10K_RXQCTL_ENABLE))
 176                         break;
 177         }
 178
 179         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 180                 return -1;
 181
 182         return 0;
 183 }
 184
 185 /*
 186  * reset queue to initial state, allocate software buffers used when starting
 187  * device
 188  */
 189 static inline void
 190 tx_queue_reset(struct fm10k_tx_queue *q)
 191 {
 192         PMD_INIT_FUNC_TRACE();
 193         q->last_free = 0;
 194         q->next_free = 0;
 195         q->nb_used = 0;
 196         q->nb_free = q->nb_desc - 1;
 197         q->free_trigger = q->nb_free - q->free_thresh;
 198         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 199         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 200 }
 201
 202 /*
 203  * clean queue, descriptor rings, free software buffers used when stopping
 204  * device
 205  */
 206 static inline void
 207 tx_queue_clean(struct fm10k_tx_queue *q)
 208 {
 209         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 210         uint32_t i;
 211         PMD_INIT_FUNC_TRACE();
 212
 213         /* zero descriptor rings */
 214         for (i = 0; i < q->nb_desc; ++i)
 215                 q->hw_ring[i] = zero;
 216
 217         /* free software buffers */
 218         for (i = 0; i < q->nb_desc; ++i) {
 219                 if (q->sw_ring[i]) {
 220                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 221                         q->sw_ring[i] = NULL;
 222                 }
 223         }
 224 }
 225
 226 /*
 227  * free all queue memory used when releasing the queue (i.e. configure)
 228  */
 229 static inline void
 230 tx_queue_free(struct fm10k_tx_queue *q)
 231 {
 232         PMD_INIT_FUNC_TRACE();
 233         if (q) {
 234                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 235                 tx_queue_clean(q);
 236                 if (q->rs_tracker.list) {
 237                         rte_free(q->rs_tracker.list);
 238                         q->rs_tracker.list = NULL;
 239                 }
 240                 if (q->sw_ring) {
 241                         rte_free(q->sw_ring);
 242                         q->sw_ring = NULL;
 243                 }
 244                 rte_free(q);
 245                 q = NULL;
 246         }
 247 }
 248
 249 /*
 250  * disable TX queue, wait unitl HW finished necessary flush operation
 251  */
 252 static inline int
 253 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 254 {
 255         uint32_t reg, i;
 256
 257         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 258         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 259                         reg & ~FM10K_TXDCTL_ENABLE);
 260
 261         /* Wait 100us at most */
 262         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 263                 rte_delay_us(1);
 264                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 265                 if (!(reg & FM10K_TXDCTL_ENABLE))
 266                         break;
 267         }
 268
 269         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 270                 return -1;
 271
 272         return 0;
 273 }
 274
 275 static int
 276 fm10k_dev_configure(struct rte_eth_dev *dev)
 277 {
 278         PMD_INIT_FUNC_TRACE();
 279
 280         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 281                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 282
 283         return 0;
 284 }
 285
 286 static void
 287 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 288 {
 289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 291         uint32_t mrqc, *key, i, reta, j;
 292         uint64_t hf;
 293
 294 #define RSS_KEY_SIZE 40
 295         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 296                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 297                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 298                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 299                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 300                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 301         };
 302
 303         if (dev->data->nb_rx_queues == 1 ||
 304             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 305             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 306                 return;
 307
 308         /* random key is rss_intel_key (default) or user provided (rss_key) */
 309         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 310                 key = (uint32_t *)rss_intel_key;
 311         else
 312                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 313
 314         /* Now fill our hash function seeds, 4 bytes at a time */
 315         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 316                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 317
 318         /*
 319          * Fill in redirection table
 320          * The byte-swap is needed because NIC registers are in
 321          * little-endian order.
 322          */
 323         reta = 0;
 324         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 325                 if (j == dev->data->nb_rx_queues)
 326                         j = 0;
 327                 reta = (reta << CHAR_BIT) | j;
 328                 if ((i & 3) == 3)
 329                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 330                                         rte_bswap32(reta));
 331         }
 332
 333         /*
 334          * Generate RSS hash based on packet types, TCP/UDP
 335          * port numbers and/or IPv4/v6 src and dst addresses
 336          */
 337         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 338         mrqc = 0;
 339         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 340         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 341         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 342         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 343         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 344         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 345         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 346         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 347         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 348
 349         if (mrqc == 0) {
 350                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 351                         "supported", hf);
 352                 return;
 353         }
 354
 355         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 356 }
 357
 358 static int
 359 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 360 {
 361         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 362         int i, ret;
 363         struct fm10k_tx_queue *txq;
 364         uint64_t base_addr;
 365         uint32_t size;
 366
 367         /* Disable TXINT to avoid possible interrupt */
 368         for (i = 0; i < hw->mac.max_queues; i++)
 369                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 370                                 3 << FM10K_TXINT_TIMER_SHIFT);
 371
 372         /* Setup TX queue */
 373         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 374                 txq = dev->data->tx_queues[i];
 375                 base_addr = txq->hw_ring_phys_addr;
 376                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 377
 378                 /* disable queue to avoid issues while updating state */
 379                 ret = tx_queue_disable(hw, i);
 380                 if (ret) {
 381                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 382                         return -1;
 383                 }
 384
 385                 /* set location and size for descriptor ring */
 386                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 387                                 base_addr & UINT64_LOWER_32BITS_MASK);
 388                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 389                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 390                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 391         }
 392         return 0;
 393 }
 394
 395 static int
 396 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 397 {
 398         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 399         int i, ret;
 400         struct fm10k_rx_queue *rxq;
 401         uint64_t base_addr;
 402         uint32_t size;
 403         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 404         uint16_t buf_size;
 405
 406         /* Disable RXINT to avoid possible interrupt */
 407         for (i = 0; i < hw->mac.max_queues; i++)
 408                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 409                                 3 << FM10K_RXINT_TIMER_SHIFT);
 410
 411         /* Setup RX queues */
 412         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 413                 rxq = dev->data->rx_queues[i];
 414                 base_addr = rxq->hw_ring_phys_addr;
 415                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 416
 417                 /* disable queue to avoid issues while updating state */
 418                 ret = rx_queue_disable(hw, i);
 419                 if (ret) {
 420                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 421                         return -1;
 422                 }
 423
 424                 /* Setup the Base and Length of the Rx Descriptor Ring */
 425                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 426                                 base_addr & UINT64_LOWER_32BITS_MASK);
 427                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 428                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 429                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 430
 431                 /* Configure the Rx buffer size for one buff without split */
 432                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 433                         RTE_PKTMBUF_HEADROOM);
 434                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 435                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 436
 437                 /* It adds dual VLAN length for supporting dual VLAN */
 438                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 439                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size){
 440                         dev->data->scattered_rx = 1;
 441                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 442                 }
 443
 444                 /* Enable drop on empty, it's RO for VF */
 445                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 446                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 447
 448                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 449                 FM10K_WRITE_FLUSH(hw);
 450         }
 451
 452         if (dev->data->dev_conf.rxmode.enable_scatter) {
 453                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 454                 dev->data->scattered_rx = 1;
 455         }
 456
 457         /* Configure RSS if applicable */
 458         fm10k_dev_mq_rx_configure(dev);
 459         return 0;
 460 }
 461
 462 static int
 463 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 464 {
 465         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 466         int err = -1;
 467         uint32_t reg;
 468         struct fm10k_rx_queue *rxq;
 469
 470         PMD_INIT_FUNC_TRACE();
 471
 472         if (rx_queue_id < dev->data->nb_rx_queues) {
 473                 rxq = dev->data->rx_queues[rx_queue_id];
 474                 err = rx_queue_reset(rxq);
 475                 if (err == -ENOMEM) {
 476                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 477                         return err;
 478                 } else if (err == -EINVAL) {
 479                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 480                                 " %d", err);
 481                         return err;
 482                 }
 483
 484                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 485                  * Note: this must be done AFTER the queue is enabled on real
 486                  * hardware, but BEFORE the queue is enabled when using the
 487                  * emulation platform. Do it in both places for now and remove
 488                  * this comment and the following two register writes when the
 489                  * emulation platform is no longer being used.
 490                  */
 491                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 492                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 493
 494                 /* Set PF ownership flag for PF devices */
 495                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 496                 if (hw->mac.type == fm10k_mac_pf)
 497                         reg |= FM10K_RXQCTL_PF;
 498                 reg |= FM10K_RXQCTL_ENABLE;
 499                 /* enable RX queue */
 500                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 501                 FM10K_WRITE_FLUSH(hw);
 502
 503                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 504                  * Note: this must be done AFTER the queue is enabled
 505                  */
 506                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 507                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 508         }
 509
 510         return err;
 511 }
 512
 513 static int
 514 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 515 {
 516         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 517
 518         PMD_INIT_FUNC_TRACE();
 519
 520         if (rx_queue_id < dev->data->nb_rx_queues) {
 521                 /* Disable RX queue */
 522                 rx_queue_disable(hw, rx_queue_id);
 523
 524                 /* Free mbuf and clean HW ring */
 525                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 526         }
 527
 528         return 0;
 529 }
 530
 531 static int
 532 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 533 {
 534         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 535         /** @todo - this should be defined in the shared code */
 536 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 537         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 538         int err = 0;
 539
 540         PMD_INIT_FUNC_TRACE();
 541
 542         if (tx_queue_id < dev->data->nb_tx_queues) {
 543                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 544
 545                 /* reset head and tail pointers */
 546                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 547                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 548
 549                 /* enable TX queue */
 550                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 551                                         FM10K_TXDCTL_ENABLE | txdctl);
 552                 FM10K_WRITE_FLUSH(hw);
 553         } else
 554                 err = -1;
 555
 556         return err;
 557 }
 558
 559 static int
 560 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 561 {
 562         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 563
 564         PMD_INIT_FUNC_TRACE();
 565
 566         if (tx_queue_id < dev->data->nb_tx_queues) {
 567                 tx_queue_disable(hw, tx_queue_id);
 568                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 569         }
 570
 571         return 0;
 572 }
 573
 574 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 575 {
 576         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 577                 != FM10K_DGLORTMAP_NONE);
 578 }
 579
 580 static void
 581 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 582 {
 583         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 584         int status;
 585
 586         PMD_INIT_FUNC_TRACE();
 587
 588         /* Return if it didn't acquire valid glort range */
 589         if (!fm10k_glort_valid(hw))
 590                 return;
 591
 592         fm10k_mbx_lock(hw);
 593         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 594                                 FM10K_XCAST_MODE_PROMISC);
 595         fm10k_mbx_unlock(hw);
 596
 597         if (status != FM10K_SUCCESS)
 598                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 599 }
 600
 601 static void
 602 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 603 {
 604         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 605         uint8_t mode;
 606         int status;
 607
 608         PMD_INIT_FUNC_TRACE();
 609
 610         /* Return if it didn't acquire valid glort range */
 611         if (!fm10k_glort_valid(hw))
 612                 return;
 613
 614         if (dev->data->all_multicast == 1)
 615                 mode = FM10K_XCAST_MODE_ALLMULTI;
 616         else
 617                 mode = FM10K_XCAST_MODE_NONE;
 618
 619         fm10k_mbx_lock(hw);
 620         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 621                                 mode);
 622         fm10k_mbx_unlock(hw);
 623
 624         if (status != FM10K_SUCCESS)
 625                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 626 }
 627
 628 static void
 629 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int status;
 633
 634         PMD_INIT_FUNC_TRACE();
 635
 636         /* Return if it didn't acquire valid glort range */
 637         if (!fm10k_glort_valid(hw))
 638                 return;
 639
 640         /* If promiscuous mode is enabled, it doesn't make sense to enable
 641          * allmulticast and disable promiscuous since fm10k only can select
 642          * one of the modes.
 643          */
 644         if (dev->data->promiscuous) {
 645                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 646                         "needn't enable allmulticast");
 647                 return;
 648         }
 649
 650         fm10k_mbx_lock(hw);
 651         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 652                                 FM10K_XCAST_MODE_ALLMULTI);
 653         fm10k_mbx_unlock(hw);
 654
 655         if (status != FM10K_SUCCESS)
 656                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 657 }
 658
 659 static void
 660 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 661 {
 662         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 663         int status;
 664
 665         PMD_INIT_FUNC_TRACE();
 666
 667         /* Return if it didn't acquire valid glort range */
 668         if (!fm10k_glort_valid(hw))
 669                 return;
 670
 671         if (dev->data->promiscuous) {
 672                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 673                         "since promisc mode is enabled");
 674                 return;
 675         }
 676
 677         fm10k_mbx_lock(hw);
 678         /* Change mode to unicast mode */
 679         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 680                                 FM10K_XCAST_MODE_NONE);
 681         fm10k_mbx_unlock(hw);
 682
 683         if (status != FM10K_SUCCESS)
 684                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 685 }
 686
 687 /* fls = find last set bit = 32 minus the number of leading zeros */
 688 #ifndef fls
 689 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 690 #endif
 691 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 692 static int
 693 fm10k_dev_start(struct rte_eth_dev *dev)
 694 {
 695         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 696         int i, diag;
 697
 698         PMD_INIT_FUNC_TRACE();
 699
 700         /* stop, init, then start the hw */
 701         diag = fm10k_stop_hw(hw);
 702         if (diag != FM10K_SUCCESS) {
 703                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 704                 return -EIO;
 705         }
 706
 707         diag = fm10k_init_hw(hw);
 708         if (diag != FM10K_SUCCESS) {
 709                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 710                 return -EIO;
 711         }
 712
 713         diag = fm10k_start_hw(hw);
 714         if (diag != FM10K_SUCCESS) {
 715                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 716                 return -EIO;
 717         }
 718
 719         diag = fm10k_dev_tx_init(dev);
 720         if (diag) {
 721                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 722                 return diag;
 723         }
 724
 725         diag = fm10k_dev_rx_init(dev);
 726         if (diag) {
 727                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 728                 return diag;
 729         }
 730
 731         if (hw->mac.type == fm10k_mac_pf) {
 732                 /* Establish only VSI 0 as valid */
 733                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 734
 735                 /* Configure RSS bits used in RETA table */
 736                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 737                                 fls(dev->data->nb_rx_queues - 1) <<
 738                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 739
 740                 /* Invalidate all other GLORT entries */
 741                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 742                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 743                                         FM10K_DGLORTMAP_NONE);
 744         }
 745
 746         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 747                 struct fm10k_rx_queue *rxq;
 748                 rxq = dev->data->rx_queues[i];
 749
 750                 if (rxq->rx_deferred_start)
 751                         continue;
 752                 diag = fm10k_dev_rx_queue_start(dev, i);
 753                 if (diag != 0) {
 754                         int j;
 755                         for (j = 0; j < i; ++j)
 756                                 rx_queue_clean(dev->data->rx_queues[j]);
 757                         return diag;
 758                 }
 759         }
 760
 761         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 762                 struct fm10k_tx_queue *txq;
 763                 txq = dev->data->tx_queues[i];
 764
 765                 if (txq->tx_deferred_start)
 766                         continue;
 767                 diag = fm10k_dev_tx_queue_start(dev, i);
 768                 if (diag != 0) {
 769                         int j;
 770                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 771                                 rx_queue_clean(dev->data->rx_queues[j]);
 772                         return diag;
 773                 }
 774         }
 775
 776         return 0;
 777 }
 778
 779 static void
 780 fm10k_dev_stop(struct rte_eth_dev *dev)
 781 {
 782         int i;
 783
 784         PMD_INIT_FUNC_TRACE();
 785
 786         for (i = 0; i < dev->data->nb_tx_queues; i++)
 787                 fm10k_dev_tx_queue_stop(dev, i);
 788
 789         for (i = 0; i < dev->data->nb_rx_queues; i++)
 790                 fm10k_dev_rx_queue_stop(dev, i);
 791 }
 792
 793 static void
 794 fm10k_dev_close(struct rte_eth_dev *dev)
 795 {
 796         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 797
 798         PMD_INIT_FUNC_TRACE();
 799
 800         /* Stop mailbox service first */
 801         fm10k_close_mbx_service(hw);
 802         fm10k_dev_stop(dev);
 803         fm10k_stop_hw(hw);
 804 }
 805
 806 static int
 807 fm10k_link_update(struct rte_eth_dev *dev,
 808         __rte_unused int wait_to_complete)
 809 {
 810         PMD_INIT_FUNC_TRACE();
 811
 812         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 813          * x8 PCIe interface. For now, we leave the speed undefined since there
 814          * is no 50Gbps Ethernet. */
 815         dev->data->dev_link.link_speed  = 0;
 816         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 817         dev->data->dev_link.link_status = 1;
 818
 819         return 0;
 820 }
 821
 822 static void
 823 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 824 {
 825         uint64_t ipackets, opackets, ibytes, obytes;
 826         struct fm10k_hw *hw =
 827                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 828         struct fm10k_hw_stats *hw_stats =
 829                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 830         int i;
 831
 832         PMD_INIT_FUNC_TRACE();
 833
 834         fm10k_update_hw_stats(hw, hw_stats);
 835
 836         ipackets = opackets = ibytes = obytes = 0;
 837         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 838                 (i < hw->mac.max_queues); ++i) {
 839                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 840                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 841                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 842                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 843                 ipackets += stats->q_ipackets[i];
 844                 opackets += stats->q_opackets[i];
 845                 ibytes   += stats->q_ibytes[i];
 846                 obytes   += stats->q_obytes[i];
 847         }
 848         stats->ipackets = ipackets;
 849         stats->opackets = opackets;
 850         stats->ibytes = ibytes;
 851         stats->obytes = obytes;
 852 }
 853
 854 static void
 855 fm10k_stats_reset(struct rte_eth_dev *dev)
 856 {
 857         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 858         struct fm10k_hw_stats *hw_stats =
 859                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 860
 861         PMD_INIT_FUNC_TRACE();
 862
 863         memset(hw_stats, 0, sizeof(*hw_stats));
 864         fm10k_rebind_hw_stats(hw, hw_stats);
 865 }
 866
 867 static void
 868 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 869         struct rte_eth_dev_info *dev_info)
 870 {
 871         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 872
 873         PMD_INIT_FUNC_TRACE();
 874
 875         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 876         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 877         dev_info->max_rx_queues      = hw->mac.max_queues;
 878         dev_info->max_tx_queues      = hw->mac.max_queues;
 879         dev_info->max_mac_addrs      = 1;
 880         dev_info->max_hash_mac_addrs = 0;
 881         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 882         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 883         dev_info->rx_offload_capa =
 884                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 885                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 886                 DEV_RX_OFFLOAD_TCP_CKSUM;
 887         dev_info->tx_offload_capa    = 0;
 888         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 889
 890         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 891                 .rx_thresh = {
 892                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 893                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 894                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 895                 },
 896                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 897                 .rx_drop_en = 0,
 898         };
 899
 900         dev_info->default_txconf = (struct rte_eth_txconf) {
 901                 .tx_thresh = {
 902                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 903                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 904                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 905                 },
 906                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 907                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 908                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 909                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 910         };
 911
 912 }
 913
 914 static int
 915 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 916 {
 917         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 918
 919         PMD_INIT_FUNC_TRACE();
 920
 921         /* @todo - add support for the VF */
 922         if (hw->mac.type != fm10k_mac_pf)
 923                 return -ENOTSUP;
 924
 925         return fm10k_update_vlan(hw, vlan_id, 0, on);
 926 }
 927
 928 static inline int
 929 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
 930 {
 931         if ((request < min) || (request > max) || ((request % mult) != 0))
 932                 return -1;
 933         else
 934                 return 0;
 935 }
 936
 937 /*
 938  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
 939  * the physical address is required. If the memzone is already created, then
 940  * this function returns a pointer to the existing memzone.
 941  */
 942 static inline const struct rte_memzone *
 943 allocate_hw_ring(const char *driver_name, const char *ring_name,
 944         uint8_t port_id, uint16_t queue_id, int socket_id,
 945         uint32_t size, uint32_t align)
 946 {
 947         char name[RTE_MEMZONE_NAMESIZE];
 948         const struct rte_memzone *mz;
 949
 950         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
 951                  driver_name, ring_name, port_id, queue_id, socket_id);
 952
 953         /* return the memzone if it already exists */
 954         mz = rte_memzone_lookup(name);
 955         if (mz)
 956                 return mz;
 957
 958 #ifdef RTE_LIBRTE_XEN_DOM0
 959         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
 960                                            RTE_PGSIZE_2M);
 961 #else
 962         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
 963 #endif
 964 }
 965
 966 static inline int
 967 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
 968 {
 969         if ((request < min) || (request > max) || ((div % request) != 0))
 970                 return -1;
 971         else
 972                 return 0;
 973 }
 974
 975 static inline int
 976 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
 977 {
 978         uint16_t rx_free_thresh;
 979
 980         if (conf->rx_free_thresh == 0)
 981                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
 982         else
 983                 rx_free_thresh = conf->rx_free_thresh;
 984
 985         /* make sure the requested threshold satisfies the constraints */
 986         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
 987                         FM10K_RX_FREE_THRESH_MAX(q),
 988                         FM10K_RX_FREE_THRESH_DIV(q),
 989                         rx_free_thresh)) {
 990                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
 991                         "less than or equal to %u, "
 992                         "greater than or equal to %u, "
 993                         "and a divisor of %u",
 994                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
 995                         FM10K_RX_FREE_THRESH_MIN(q),
 996                         FM10K_RX_FREE_THRESH_DIV(q));
 997                 return (-EINVAL);
 998         }
 999
1000         q->alloc_thresh = rx_free_thresh;
1001         q->drop_en = conf->rx_drop_en;
1002         q->rx_deferred_start = conf->rx_deferred_start;
1003
1004         return 0;
1005 }
1006
1007 /*
1008  * Hardware requires specific alignment for Rx packet buffers. At
1009  * least one of the following two conditions must be satisfied.
1010  *  1. Address is 512B aligned
1011  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1012  *
1013  * As such, the driver may need to adjust the DMA address within the
1014  * buffer by up to 512B.
1015  *
1016  * return 1 if the element size is valid, otherwise return 0.
1017  */
1018 static int
1019 mempool_element_size_valid(struct rte_mempool *mp)
1020 {
1021         uint32_t min_size;
1022
1023         /* elt_size includes mbuf header and headroom */
1024         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1025                         RTE_PKTMBUF_HEADROOM;
1026
1027         /* account for up to 512B of alignment */
1028         min_size -= FM10K_RX_BUFF_ALIGN;
1029
1030         /* sanity check for overflow */
1031         if (min_size > mp->elt_size)
1032                 return 0;
1033
1034         /* size is valid */
1035         return 1;
1036 }
1037
1038 static int
1039 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1040         uint16_t nb_desc, unsigned int socket_id,
1041         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1042 {
1043         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1044         struct fm10k_rx_queue *q;
1045         const struct rte_memzone *mz;
1046
1047         PMD_INIT_FUNC_TRACE();
1048
1049         /* make sure the mempool element size can account for alignment. */
1050         if (!mempool_element_size_valid(mp)) {
1051                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1052                 return (-EINVAL);
1053         }
1054
1055         /* make sure a valid number of descriptors have been requested */
1056         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1057                                 FM10K_MULT_RX_DESC, nb_desc)) {
1058                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1059                         "less than or equal to %"PRIu32", "
1060                         "greater than or equal to %u, "
1061                         "and a multiple of %u",
1062                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1063                         FM10K_MULT_RX_DESC);
1064                 return (-EINVAL);
1065         }
1066
1067         /*
1068          * if this queue existed already, free the associated memory. The
1069          * queue cannot be reused in case we need to allocate memory on
1070          * different socket than was previously used.
1071          */
1072         if (dev->data->rx_queues[queue_id] != NULL) {
1073                 rx_queue_free(dev->data->rx_queues[queue_id]);
1074                 dev->data->rx_queues[queue_id] = NULL;
1075         }
1076
1077         /* allocate memory for the queue structure */
1078         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1079                                 socket_id);
1080         if (q == NULL) {
1081                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1082                 return (-ENOMEM);
1083         }
1084
1085         /* setup queue */
1086         q->mp = mp;
1087         q->nb_desc = nb_desc;
1088         q->port_id = dev->data->port_id;
1089         q->queue_id = queue_id;
1090         q->tail_ptr = (volatile uint32_t *)
1091                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1092         if (handle_rxconf(q, conf))
1093                 return (-EINVAL);
1094
1095         /* allocate memory for the software ring */
1096         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1097                                         nb_desc * sizeof(struct rte_mbuf *),
1098                                         RTE_CACHE_LINE_SIZE, socket_id);
1099         if (q->sw_ring == NULL) {
1100                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1101                 rte_free(q);
1102                 return (-ENOMEM);
1103         }
1104
1105         /*
1106          * allocate memory for the hardware descriptor ring. A memzone large
1107          * enough to hold the maximum ring size is requested to allow for
1108          * resizing in later calls to the queue setup function.
1109          */
1110         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1111                                 dev->data->port_id, queue_id, socket_id,
1112                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1113         if (mz == NULL) {
1114                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1115                 rte_free(q->sw_ring);
1116                 rte_free(q);
1117                 return (-ENOMEM);
1118         }
1119         q->hw_ring = mz->addr;
1120         q->hw_ring_phys_addr = mz->phys_addr;
1121
1122         dev->data->rx_queues[queue_id] = q;
1123         return 0;
1124 }
1125
1126 static void
1127 fm10k_rx_queue_release(void *queue)
1128 {
1129         PMD_INIT_FUNC_TRACE();
1130
1131         rx_queue_free(queue);
1132 }
1133
1134 static inline int
1135 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1136 {
1137         uint16_t tx_free_thresh;
1138         uint16_t tx_rs_thresh;
1139
1140         /* constraint MACROs require that tx_free_thresh is configured
1141          * before tx_rs_thresh */
1142         if (conf->tx_free_thresh == 0)
1143                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1144         else
1145                 tx_free_thresh = conf->tx_free_thresh;
1146
1147         /* make sure the requested threshold satisfies the constraints */
1148         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1149                         FM10K_TX_FREE_THRESH_MAX(q),
1150                         FM10K_TX_FREE_THRESH_DIV(q),
1151                         tx_free_thresh)) {
1152                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1153                         "less than or equal to %u, "
1154                         "greater than or equal to %u, "
1155                         "and a divisor of %u",
1156                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1157                         FM10K_TX_FREE_THRESH_MIN(q),
1158                         FM10K_TX_FREE_THRESH_DIV(q));
1159                 return (-EINVAL);
1160         }
1161
1162         q->free_thresh = tx_free_thresh;
1163
1164         if (conf->tx_rs_thresh == 0)
1165                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1166         else
1167                 tx_rs_thresh = conf->tx_rs_thresh;
1168
1169         q->tx_deferred_start = conf->tx_deferred_start;
1170
1171         /* make sure the requested threshold satisfies the constraints */
1172         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1173                         FM10K_TX_RS_THRESH_MAX(q),
1174                         FM10K_TX_RS_THRESH_DIV(q),
1175                         tx_rs_thresh)) {
1176                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1177                         "less than or equal to %u, "
1178                         "greater than or equal to %u, "
1179                         "and a divisor of %u",
1180                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1181                         FM10K_TX_RS_THRESH_MIN(q),
1182                         FM10K_TX_RS_THRESH_DIV(q));
1183                 return (-EINVAL);
1184         }
1185
1186         q->rs_thresh = tx_rs_thresh;
1187
1188         return 0;
1189 }
1190
1191 static int
1192 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1193         uint16_t nb_desc, unsigned int socket_id,
1194         const struct rte_eth_txconf *conf)
1195 {
1196         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1197         struct fm10k_tx_queue *q;
1198         const struct rte_memzone *mz;
1199
1200         PMD_INIT_FUNC_TRACE();
1201
1202         /* make sure a valid number of descriptors have been requested */
1203         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1204                                 FM10K_MULT_TX_DESC, nb_desc)) {
1205                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1206                         "less than or equal to %"PRIu32", "
1207                         "greater than or equal to %u, "
1208                         "and a multiple of %u",
1209                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1210                         FM10K_MULT_TX_DESC);
1211                 return (-EINVAL);
1212         }
1213
1214         /*
1215          * if this queue existed already, free the associated memory. The
1216          * queue cannot be reused in case we need to allocate memory on
1217          * different socket than was previously used.
1218          */
1219         if (dev->data->tx_queues[queue_id] != NULL) {
1220                 tx_queue_free(dev->data->tx_queues[queue_id]);
1221                 dev->data->tx_queues[queue_id] = NULL;
1222         }
1223
1224         /* allocate memory for the queue structure */
1225         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1226                                 socket_id);
1227         if (q == NULL) {
1228                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1229                 return (-ENOMEM);
1230         }
1231
1232         /* setup queue */
1233         q->nb_desc = nb_desc;
1234         q->port_id = dev->data->port_id;
1235         q->queue_id = queue_id;
1236         q->tail_ptr = (volatile uint32_t *)
1237                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1238         if (handle_txconf(q, conf))
1239                 return (-EINVAL);
1240
1241         /* allocate memory for the software ring */
1242         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1243                                         nb_desc * sizeof(struct rte_mbuf *),
1244                                         RTE_CACHE_LINE_SIZE, socket_id);
1245         if (q->sw_ring == NULL) {
1246                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1247                 rte_free(q);
1248                 return (-ENOMEM);
1249         }
1250
1251         /*
1252          * allocate memory for the hardware descriptor ring. A memzone large
1253          * enough to hold the maximum ring size is requested to allow for
1254          * resizing in later calls to the queue setup function.
1255          */
1256         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1257                                 dev->data->port_id, queue_id, socket_id,
1258                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1259         if (mz == NULL) {
1260                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1261                 rte_free(q->sw_ring);
1262                 rte_free(q);
1263                 return (-ENOMEM);
1264         }
1265         q->hw_ring = mz->addr;
1266         q->hw_ring_phys_addr = mz->phys_addr;
1267
1268         /*
1269          * allocate memory for the RS bit tracker. Enough slots to hold the
1270          * descriptor index for each RS bit needing to be set are required.
1271          */
1272         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1273                                 ((nb_desc + 1) / q->rs_thresh) *
1274                                 sizeof(uint16_t),
1275                                 RTE_CACHE_LINE_SIZE, socket_id);
1276         if (q->rs_tracker.list == NULL) {
1277                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1278                 rte_free(q->sw_ring);
1279                 rte_free(q);
1280                 return (-ENOMEM);
1281         }
1282
1283         dev->data->tx_queues[queue_id] = q;
1284         return 0;
1285 }
1286
1287 static void
1288 fm10k_tx_queue_release(void *queue)
1289 {
1290         PMD_INIT_FUNC_TRACE();
1291
1292         tx_queue_free(queue);
1293 }
1294
1295 static int
1296 fm10k_reta_update(struct rte_eth_dev *dev,
1297                         struct rte_eth_rss_reta_entry64 *reta_conf,
1298                         uint16_t reta_size)
1299 {
1300         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1301         uint16_t i, j, idx, shift;
1302         uint8_t mask;
1303         uint32_t reta;
1304
1305         PMD_INIT_FUNC_TRACE();
1306
1307         if (reta_size > FM10K_MAX_RSS_INDICES) {
1308                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1309                         "(%d) doesn't match the number hardware can supported "
1310                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1311                 return -EINVAL;
1312         }
1313
1314         /*
1315          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1316          * 128-entries in 32 registers
1317          */
1318         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1319                 idx = i / RTE_RETA_GROUP_SIZE;
1320                 shift = i % RTE_RETA_GROUP_SIZE;
1321                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1322                                 BIT_MASK_PER_UINT32);
1323                 if (mask == 0)
1324                         continue;
1325
1326                 reta = 0;
1327                 if (mask != BIT_MASK_PER_UINT32)
1328                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1329
1330                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1331                         if (mask & (0x1 << j)) {
1332                                 if (mask != 0xF)
1333                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1334                                 reta |= reta_conf[idx].reta[shift + j] <<
1335                                                 (CHAR_BIT * j);
1336                         }
1337                 }
1338                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1339         }
1340
1341         return 0;
1342 }
1343
1344 static int
1345 fm10k_reta_query(struct rte_eth_dev *dev,
1346                         struct rte_eth_rss_reta_entry64 *reta_conf,
1347                         uint16_t reta_size)
1348 {
1349         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1350         uint16_t i, j, idx, shift;
1351         uint8_t mask;
1352         uint32_t reta;
1353
1354         PMD_INIT_FUNC_TRACE();
1355
1356         if (reta_size < FM10K_MAX_RSS_INDICES) {
1357                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1358                         "(%d) doesn't match the number hardware can supported "
1359                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1360                 return -EINVAL;
1361         }
1362
1363         /*
1364          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1365          * 128-entries in 32 registers
1366          */
1367         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1368                 idx = i / RTE_RETA_GROUP_SIZE;
1369                 shift = i % RTE_RETA_GROUP_SIZE;
1370                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1371                                 BIT_MASK_PER_UINT32);
1372                 if (mask == 0)
1373                         continue;
1374
1375                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1376                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1377                         if (mask & (0x1 << j))
1378                                 reta_conf[idx].reta[shift + j] = ((reta >>
1379                                         CHAR_BIT * j) & UINT8_MAX);
1380                 }
1381         }
1382
1383         return 0;
1384 }
1385
1386 static int
1387 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1388         struct rte_eth_rss_conf *rss_conf)
1389 {
1390         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1391         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1392         uint32_t mrqc;
1393         uint64_t hf = rss_conf->rss_hf;
1394         int i;
1395
1396         PMD_INIT_FUNC_TRACE();
1397
1398         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1399                 FM10K_RSSRK_ENTRIES_PER_REG)
1400                 return -EINVAL;
1401
1402         if (hf == 0)
1403                 return -EINVAL;
1404
1405         mrqc = 0;
1406         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1407         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1408         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1409         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1410         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1411         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1412         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1413         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1414         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1415
1416         /* If the mapping doesn't fit any supported, return */
1417         if (mrqc == 0)
1418                 return -EINVAL;
1419
1420         if (key != NULL)
1421                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1422                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1423
1424         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1425
1426         return 0;
1427 }
1428
1429 static int
1430 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1431         struct rte_eth_rss_conf *rss_conf)
1432 {
1433         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1434         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1435         uint32_t mrqc;
1436         uint64_t hf;
1437         int i;
1438
1439         PMD_INIT_FUNC_TRACE();
1440
1441         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1442                                 FM10K_RSSRK_ENTRIES_PER_REG)
1443                 return -EINVAL;
1444
1445         if (key != NULL)
1446                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1447                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1448
1449         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1450         hf = 0;
1451         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1452         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1453         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1454         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1455         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1456         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1457         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1458         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1459         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1460
1461         rss_conf->rss_hf = hf;
1462
1463         return 0;
1464 }
1465
1466 static void
1467 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1468 {
1469         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1470         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1471
1472         /* Bind all local non-queue interrupt to vector 0 */
1473         int_map |= 0;
1474
1475         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1476         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1477         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1478         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1479         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1480         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1481
1482         /* Enable misc causes */
1483         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1484                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1485                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1486                                 FM10K_EIMR_ENABLE(MAILBOX) |
1487                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1488                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1489                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1490                                 FM10K_EIMR_ENABLE(VFLR));
1491
1492         /* Enable ITR 0 */
1493         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1494                                         FM10K_ITR_MASK_CLEAR);
1495         FM10K_WRITE_FLUSH(hw);
1496 }
1497
1498 static void
1499 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1500 {
1501         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1502         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1503
1504         /* Bind all local non-queue interrupt to vector 0 */
1505         int_map |= 0;
1506
1507         /* Only INT 0 available, other 15 are reserved. */
1508         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1509
1510         /* Enable ITR 0 */
1511         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1512                                         FM10K_ITR_MASK_CLEAR);
1513         FM10K_WRITE_FLUSH(hw);
1514 }
1515
1516 static int
1517 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1518 {
1519         struct fm10k_fault fault;
1520         int err;
1521         const char *estr = "Unknown error";
1522
1523         /* Process PCA fault */
1524         if (eicr & FM10K_EIMR_PCA_FAULT) {
1525                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1526                 if (err)
1527                         goto error;
1528                 switch (fault.type) {
1529                 case PCA_NO_FAULT:
1530                         estr = "PCA_NO_FAULT"; break;
1531                 case PCA_UNMAPPED_ADDR:
1532                         estr = "PCA_UNMAPPED_ADDR"; break;
1533                 case PCA_BAD_QACCESS_PF:
1534                         estr = "PCA_BAD_QACCESS_PF"; break;
1535                 case PCA_BAD_QACCESS_VF:
1536                         estr = "PCA_BAD_QACCESS_VF"; break;
1537                 case PCA_MALICIOUS_REQ:
1538                         estr = "PCA_MALICIOUS_REQ"; break;
1539                 case PCA_POISONED_TLP:
1540                         estr = "PCA_POISONED_TLP"; break;
1541                 case PCA_TLP_ABORT:
1542                         estr = "PCA_TLP_ABORT"; break;
1543                 default:
1544                         goto error;
1545                 }
1546                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1547                         estr, fault.func ? "VF" : "PF", fault.func,
1548                         fault.address, fault.specinfo);
1549         }
1550
1551         /* Process THI fault */
1552         if (eicr & FM10K_EIMR_THI_FAULT) {
1553                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1554                 if (err)
1555                         goto error;
1556                 switch (fault.type) {
1557                 case THI_NO_FAULT:
1558                         estr = "THI_NO_FAULT"; break;
1559                 case THI_MAL_DIS_Q_FAULT:
1560                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1561                 default:
1562                         goto error;
1563                 }
1564                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1565                         estr, fault.func ? "VF" : "PF", fault.func,
1566                         fault.address, fault.specinfo);
1567         }
1568
1569         /* Process FUM fault */
1570         if (eicr & FM10K_EIMR_FUM_FAULT) {
1571                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1572                 if (err)
1573                         goto error;
1574                 switch (fault.type) {
1575                 case FUM_NO_FAULT:
1576                         estr = "FUM_NO_FAULT"; break;
1577                 case FUM_UNMAPPED_ADDR:
1578                         estr = "FUM_UNMAPPED_ADDR"; break;
1579                 case FUM_POISONED_TLP:
1580                         estr = "FUM_POISONED_TLP"; break;
1581                 case FUM_BAD_VF_QACCESS:
1582                         estr = "FUM_BAD_VF_QACCESS"; break;
1583                 case FUM_ADD_DECODE_ERR:
1584                         estr = "FUM_ADD_DECODE_ERR"; break;
1585                 case FUM_RO_ERROR:
1586                         estr = "FUM_RO_ERROR"; break;
1587                 case FUM_QPRC_CRC_ERROR:
1588                         estr = "FUM_QPRC_CRC_ERROR"; break;
1589                 case FUM_CSR_TIMEOUT:
1590                         estr = "FUM_CSR_TIMEOUT"; break;
1591                 case FUM_INVALID_TYPE:
1592                         estr = "FUM_INVALID_TYPE"; break;
1593                 case FUM_INVALID_LENGTH:
1594                         estr = "FUM_INVALID_LENGTH"; break;
1595                 case FUM_INVALID_BE:
1596                         estr = "FUM_INVALID_BE"; break;
1597                 case FUM_INVALID_ALIGN:
1598                         estr = "FUM_INVALID_ALIGN"; break;
1599                 default:
1600                         goto error;
1601                 }
1602                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1603                         estr, fault.func ? "VF" : "PF", fault.func,
1604                         fault.address, fault.specinfo);
1605         }
1606
1607         if (estr)
1608                 return 0;
1609         return 0;
1610 error:
1611         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1612         return err;
1613 }
1614
1615 /**
1616  * PF interrupt handler triggered by NIC for handling specific interrupt.
1617  *
1618  * @param handle
1619  *  Pointer to interrupt handle.
1620  * @param param
1621  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1622  *
1623  * @return
1624  *  void
1625  */
1626 static void
1627 fm10k_dev_interrupt_handler_pf(
1628                         __rte_unused struct rte_intr_handle *handle,
1629                         void *param)
1630 {
1631         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1632         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1633         uint32_t cause, status;
1634
1635         if (hw->mac.type != fm10k_mac_pf)
1636                 return;
1637
1638         cause = FM10K_READ_REG(hw, FM10K_EICR);
1639
1640         /* Handle PCI fault cases */
1641         if (cause & FM10K_EICR_FAULT_MASK) {
1642                 PMD_INIT_LOG(ERR, "INT: find fault!");
1643                 fm10k_dev_handle_fault(hw, cause);
1644         }
1645
1646         /* Handle switch up/down */
1647         if (cause & FM10K_EICR_SWITCHNOTREADY)
1648                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1649
1650         if (cause & FM10K_EICR_SWITCHREADY)
1651                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1652
1653         /* Handle mailbox message */
1654         fm10k_mbx_lock(hw);
1655         hw->mbx.ops.process(hw, &hw->mbx);
1656         fm10k_mbx_unlock(hw);
1657
1658         /* Handle SRAM error */
1659         if (cause & FM10K_EICR_SRAMERROR) {
1660                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1661
1662                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1663                 /* Write to clear pending bits */
1664                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1665
1666                 /* Todo: print out error message after shared code  updates */
1667         }
1668
1669         /* Clear these 3 events if having any */
1670         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1671                  FM10K_EICR_SWITCHREADY;
1672         if (cause)
1673                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1674
1675         /* Re-enable interrupt from device side */
1676         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1677                                         FM10K_ITR_MASK_CLEAR);
1678         /* Re-enable interrupt from host side */
1679         rte_intr_enable(&(dev->pci_dev->intr_handle));
1680 }
1681
1682 /**
1683  * VF interrupt handler triggered by NIC for handling specific interrupt.
1684  *
1685  * @param handle
1686  *  Pointer to interrupt handle.
1687  * @param param
1688  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1689  *
1690  * @return
1691  *  void
1692  */
1693 static void
1694 fm10k_dev_interrupt_handler_vf(
1695                         __rte_unused struct rte_intr_handle *handle,
1696                         void *param)
1697 {
1698         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1699         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1700
1701         if (hw->mac.type != fm10k_mac_vf)
1702                 return;
1703
1704         /* Handle mailbox message if lock is acquired */
1705         fm10k_mbx_lock(hw);
1706         hw->mbx.ops.process(hw, &hw->mbx);
1707         fm10k_mbx_unlock(hw);
1708
1709         /* Re-enable interrupt from device side */
1710         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1711                                         FM10K_ITR_MASK_CLEAR);
1712         /* Re-enable interrupt from host side */
1713         rte_intr_enable(&(dev->pci_dev->intr_handle));
1714 }
1715
1716 /* Mailbox message handler in VF */
1717 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1718         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1719         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1720         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1721         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1722 };
1723
1724 /* Mailbox message handler in PF */
1725 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1726         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1727         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1728         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1729         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1730         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1731         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1732         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1733 };
1734
1735 static int
1736 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1737 {
1738         int err;
1739
1740         /* Initialize mailbox lock */
1741         fm10k_mbx_initlock(hw);
1742
1743         /* Replace default message handler with new ones */
1744         if (hw->mac.type == fm10k_mac_pf)
1745                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1746         else
1747                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1748
1749         if (err) {
1750                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1751                                 err);
1752                 return err;
1753         }
1754         /* Connect to SM for PF device or PF for VF device */
1755         return hw->mbx.ops.connect(hw, &hw->mbx);
1756 }
1757
1758 static void
1759 fm10k_close_mbx_service(struct fm10k_hw *hw)
1760 {
1761         /* Disconnect from SM for PF device or PF for VF device */
1762         hw->mbx.ops.disconnect(hw, &hw->mbx);
1763 }
1764
1765 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1766         .dev_configure          = fm10k_dev_configure,
1767         .dev_start              = fm10k_dev_start,
1768         .dev_stop               = fm10k_dev_stop,
1769         .dev_close              = fm10k_dev_close,
1770         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
1771         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
1772         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
1773         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
1774         .stats_get              = fm10k_stats_get,
1775         .stats_reset            = fm10k_stats_reset,
1776         .link_update            = fm10k_link_update,
1777         .dev_infos_get          = fm10k_dev_infos_get,
1778         .vlan_filter_set        = fm10k_vlan_filter_set,
1779         .rx_queue_start         = fm10k_dev_rx_queue_start,
1780         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1781         .tx_queue_start         = fm10k_dev_tx_queue_start,
1782         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1783         .rx_queue_setup         = fm10k_rx_queue_setup,
1784         .rx_queue_release       = fm10k_rx_queue_release,
1785         .tx_queue_setup         = fm10k_tx_queue_setup,
1786         .tx_queue_release       = fm10k_tx_queue_release,
1787         .reta_update            = fm10k_reta_update,
1788         .reta_query             = fm10k_reta_query,
1789         .rss_hash_update        = fm10k_rss_hash_update,
1790         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1791 };
1792
1793 static int
1794 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1795 {
1796         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1797         int diag;
1798
1799         PMD_INIT_FUNC_TRACE();
1800
1801         dev->dev_ops = &fm10k_eth_dev_ops;
1802         dev->rx_pkt_burst = &fm10k_recv_pkts;
1803         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1804
1805         if (dev->data->scattered_rx)
1806                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1807
1808         /* only initialize in the primary process */
1809         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1810                 return 0;
1811
1812         /* Vendor and Device ID need to be set before init of shared code */
1813         memset(hw, 0, sizeof(*hw));
1814         hw->device_id = dev->pci_dev->id.device_id;
1815         hw->vendor_id = dev->pci_dev->id.vendor_id;
1816         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1817         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1818         hw->revision_id = 0;
1819         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1820         if (hw->hw_addr == NULL) {
1821                 PMD_INIT_LOG(ERR, "Bad mem resource."
1822                         " Try to blacklist unused devices.");
1823                 return -EIO;
1824         }
1825
1826         /* Store fm10k_adapter pointer */
1827         hw->back = dev->data->dev_private;
1828
1829         /* Initialize the shared code */
1830         diag = fm10k_init_shared_code(hw);
1831         if (diag != FM10K_SUCCESS) {
1832                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1833                 return -EIO;
1834         }
1835
1836         /*
1837          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1838          * there is no way to get link status without reading BAR4.  Until this
1839          * works, assume we have maximum bandwidth.
1840          * @todo - fix bus info
1841          */
1842         hw->bus_caps.speed = fm10k_bus_speed_8000;
1843         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1844         hw->bus_caps.payload = fm10k_bus_payload_512;
1845         hw->bus.speed = fm10k_bus_speed_8000;
1846         hw->bus.width = fm10k_bus_width_pcie_x8;
1847         hw->bus.payload = fm10k_bus_payload_256;
1848
1849         /* Initialize the hw */
1850         diag = fm10k_init_hw(hw);
1851         if (diag != FM10K_SUCCESS) {
1852                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1853                 return -EIO;
1854         }
1855
1856         /* Initialize MAC address(es) */
1857         dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1858         if (dev->data->mac_addrs == NULL) {
1859                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1860                 return -ENOMEM;
1861         }
1862
1863         diag = fm10k_read_mac_addr(hw);
1864         if (diag != FM10K_SUCCESS) {
1865                 /*
1866                  * TODO: remove special handling on VF. Need shared code to
1867                  * fix first.
1868                  */
1869                 if (hw->mac.type == fm10k_mac_pf) {
1870                         PMD_INIT_LOG(ERR, "Read MAC addr failed: %d", diag);
1871                         return -EIO;
1872                 } else {
1873                         /* Generate a random addr */
1874                         eth_random_addr(hw->mac.addr);
1875                         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1876                 }
1877         }
1878
1879         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1880                         &dev->data->mac_addrs[0]);
1881
1882         /* Reset the hw statistics */
1883         fm10k_stats_reset(dev);
1884
1885         /* Reset the hw */
1886         diag = fm10k_reset_hw(hw);
1887         if (diag != FM10K_SUCCESS) {
1888                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1889                 return -EIO;
1890         }
1891
1892         /* Setup mailbox service */
1893         diag = fm10k_setup_mbx_service(hw);
1894         if (diag != FM10K_SUCCESS) {
1895                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1896                 return -EIO;
1897         }
1898
1899         /*PF/VF has different interrupt handling mechanism */
1900         if (hw->mac.type == fm10k_mac_pf) {
1901                 /* register callback func to eal lib */
1902                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1903                         fm10k_dev_interrupt_handler_pf, (void *)dev);
1904
1905                 /* enable MISC interrupt */
1906                 fm10k_dev_enable_intr_pf(dev);
1907         } else { /* VF */
1908                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1909                         fm10k_dev_interrupt_handler_vf, (void *)dev);
1910
1911                 fm10k_dev_enable_intr_vf(dev);
1912         }
1913
1914         /*
1915          * Below function will trigger operations on mailbox, acquire lock to
1916          * avoid race condition from interrupt handler. Operations on mailbox
1917          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
1918          * will handle and generate an interrupt to our side. Then,  FIFO in
1919          * mailbox will be touched.
1920          */
1921         fm10k_mbx_lock(hw);
1922         /* Enable port first */
1923         hw->mac.ops.update_lport_state(hw, 0, 0, 1);
1924
1925         /* Update default vlan */
1926         hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
1927
1928         /*
1929          * Add default mac/vlan filter. glort is assigned by SM for PF, while is
1930          * unused for VF. PF will assign correct glort for VF.
1931          */
1932         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
1933                               hw->mac.default_vid, 1, 0);
1934
1935         /* Set unicast mode by default. App can change to other mode in other
1936          * API func.
1937          */
1938         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1939                                         FM10K_XCAST_MODE_NONE);
1940
1941         fm10k_mbx_unlock(hw);
1942
1943         /* enable uio intr after callback registered */
1944         rte_intr_enable(&(dev->pci_dev->intr_handle));
1945
1946         return 0;
1947 }
1948
1949 /*
1950  * The set of PCI devices this driver supports. This driver will enable both PF
1951  * and SRIOV-VF devices.
1952  */
1953 static const struct rte_pci_id pci_id_fm10k_map[] = {
1954 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1955 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1956 #include "rte_pci_dev_ids.h"
1957         { .vendor_id = 0, /* sentinel */ },
1958 };
1959
1960 static struct eth_driver rte_pmd_fm10k = {
1961         .pci_drv = {
1962                 .name = "rte_pmd_fm10k",
1963                 .id_table = pci_id_fm10k_map,
1964                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1965         },
1966         .eth_dev_init = eth_fm10k_dev_init,
1967         .dev_private_size = sizeof(struct fm10k_adapter),
1968 };
1969
1970 /*
1971  * Driver initialization routine.
1972  * Invoked once at EAL init time.
1973  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
1974  */
1975 static int
1976 rte_pmd_fm10k_init(__rte_unused const char *name,
1977         __rte_unused const char *params)
1978 {
1979         PMD_INIT_FUNC_TRACE();
1980         rte_eth_driver_register(&rte_pmd_fm10k);
1981         return 0;
1982 }
1983
1984 static struct rte_driver rte_fm10k_driver = {
1985         .type = PMD_PDEV,
1986         .init = rte_pmd_fm10k_init,
1987 };
1988
1989 PMD_REGISTER_DRIVER(rte_fm10k_driver);