drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 #define FM10K_RX_BUFF_ALIGN 512
  45 /* Default delay to acquire mailbox lock */
  46 #define FM10K_MBXLOCK_DELAY_US 20
  47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  48
  49 /* Number of chars per uint32 type */
  50 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  51 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  52
  53 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  54 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  55 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  56 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  57 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  58 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  59
  60 static void
  61 fm10k_mbx_initlock(struct fm10k_hw *hw)
  62 {
  63         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  64 }
  65
  66 static void
  67 fm10k_mbx_lock(struct fm10k_hw *hw)
  68 {
  69         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  70                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  71 }
  72
  73 static void
  74 fm10k_mbx_unlock(struct fm10k_hw *hw)
  75 {
  76         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  77 }
  78
  79 /*
  80  * reset queue to initial state, allocate software buffers used when starting
  81  * device.
  82  * return 0 on success
  83  * return -ENOMEM if buffers cannot be allocated
  84  * return -EINVAL if buffers do not satisfy alignment condition
  85  */
  86 static inline int
  87 rx_queue_reset(struct fm10k_rx_queue *q)
  88 {
  89         uint64_t dma_addr;
  90         int i, diag;
  91         PMD_INIT_FUNC_TRACE();
  92
  93         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
  94         if (diag != 0)
  95                 return -ENOMEM;
  96
  97         for (i = 0; i < q->nb_desc; ++i) {
  98                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
  99                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 100                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 101                                                 q->nb_desc);
 102                         return -EINVAL;
 103                 }
 104                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 105                 q->hw_ring[i].q.pkt_addr = dma_addr;
 106                 q->hw_ring[i].q.hdr_addr = dma_addr;
 107         }
 108
 109         q->next_dd = 0;
 110         q->next_alloc = 0;
 111         q->next_trigger = q->alloc_thresh - 1;
 112         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 113         return 0;
 114 }
 115
 116 /*
 117  * clean queue, descriptor rings, free software buffers used when stopping
 118  * device.
 119  */
 120 static inline void
 121 rx_queue_clean(struct fm10k_rx_queue *q)
 122 {
 123         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 124         uint32_t i;
 125         PMD_INIT_FUNC_TRACE();
 126
 127         /* zero descriptor rings */
 128         for (i = 0; i < q->nb_desc; ++i)
 129                 q->hw_ring[i] = zero;
 130
 131         /* free software buffers */
 132         for (i = 0; i < q->nb_desc; ++i) {
 133                 if (q->sw_ring[i]) {
 134                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 135                         q->sw_ring[i] = NULL;
 136                 }
 137         }
 138 }
 139
 140 /*
 141  * free all queue memory used when releasing the queue (i.e. configure)
 142  */
 143 static inline void
 144 rx_queue_free(struct fm10k_rx_queue *q)
 145 {
 146         PMD_INIT_FUNC_TRACE();
 147         if (q) {
 148                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 149                 rx_queue_clean(q);
 150                 if (q->sw_ring) {
 151                         rte_free(q->sw_ring);
 152                         q->sw_ring = NULL;
 153                 }
 154                 rte_free(q);
 155                 q = NULL;
 156         }
 157 }
 158
 159 /*
 160  * disable RX queue, wait unitl HW finished necessary flush operation
 161  */
 162 static inline int
 163 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 164 {
 165         uint32_t reg, i;
 166
 167         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 168         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 169                         reg & ~FM10K_RXQCTL_ENABLE);
 170
 171         /* Wait 100us at most */
 172         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 173                 rte_delay_us(1);
 174                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 175                 if (!(reg & FM10K_RXQCTL_ENABLE))
 176                         break;
 177         }
 178
 179         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 180                 return -1;
 181
 182         return 0;
 183 }
 184
 185 /*
 186  * reset queue to initial state, allocate software buffers used when starting
 187  * device
 188  */
 189 static inline void
 190 tx_queue_reset(struct fm10k_tx_queue *q)
 191 {
 192         PMD_INIT_FUNC_TRACE();
 193         q->last_free = 0;
 194         q->next_free = 0;
 195         q->nb_used = 0;
 196         q->nb_free = q->nb_desc - 1;
 197         q->free_trigger = q->nb_free - q->free_thresh;
 198         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 199         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 200 }
 201
 202 /*
 203  * clean queue, descriptor rings, free software buffers used when stopping
 204  * device
 205  */
 206 static inline void
 207 tx_queue_clean(struct fm10k_tx_queue *q)
 208 {
 209         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 210         uint32_t i;
 211         PMD_INIT_FUNC_TRACE();
 212
 213         /* zero descriptor rings */
 214         for (i = 0; i < q->nb_desc; ++i)
 215                 q->hw_ring[i] = zero;
 216
 217         /* free software buffers */
 218         for (i = 0; i < q->nb_desc; ++i) {
 219                 if (q->sw_ring[i]) {
 220                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 221                         q->sw_ring[i] = NULL;
 222                 }
 223         }
 224 }
 225
 226 /*
 227  * free all queue memory used when releasing the queue (i.e. configure)
 228  */
 229 static inline void
 230 tx_queue_free(struct fm10k_tx_queue *q)
 231 {
 232         PMD_INIT_FUNC_TRACE();
 233         if (q) {
 234                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 235                 tx_queue_clean(q);
 236                 if (q->rs_tracker.list) {
 237                         rte_free(q->rs_tracker.list);
 238                         q->rs_tracker.list = NULL;
 239                 }
 240                 if (q->sw_ring) {
 241                         rte_free(q->sw_ring);
 242                         q->sw_ring = NULL;
 243                 }
 244                 rte_free(q);
 245                 q = NULL;
 246         }
 247 }
 248
 249 /*
 250  * disable TX queue, wait unitl HW finished necessary flush operation
 251  */
 252 static inline int
 253 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 254 {
 255         uint32_t reg, i;
 256
 257         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 258         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 259                         reg & ~FM10K_TXDCTL_ENABLE);
 260
 261         /* Wait 100us at most */
 262         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 263                 rte_delay_us(1);
 264                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 265                 if (!(reg & FM10K_TXDCTL_ENABLE))
 266                         break;
 267         }
 268
 269         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 270                 return -1;
 271
 272         return 0;
 273 }
 274
 275 static int
 276 fm10k_dev_configure(struct rte_eth_dev *dev)
 277 {
 278         PMD_INIT_FUNC_TRACE();
 279
 280         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 281                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 282
 283         return 0;
 284 }
 285
 286 static void
 287 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 288 {
 289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 291         uint32_t mrqc, *key, i, reta, j;
 292         uint64_t hf;
 293
 294 #define RSS_KEY_SIZE 40
 295         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 296                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 297                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 298                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 299                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 300                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 301         };
 302
 303         if (dev->data->nb_rx_queues == 1 ||
 304             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 305             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 306                 return;
 307
 308         /* random key is rss_intel_key (default) or user provided (rss_key) */
 309         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 310                 key = (uint32_t *)rss_intel_key;
 311         else
 312                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 313
 314         /* Now fill our hash function seeds, 4 bytes at a time */
 315         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 316                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 317
 318         /*
 319          * Fill in redirection table
 320          * The byte-swap is needed because NIC registers are in
 321          * little-endian order.
 322          */
 323         reta = 0;
 324         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 325                 if (j == dev->data->nb_rx_queues)
 326                         j = 0;
 327                 reta = (reta << CHAR_BIT) | j;
 328                 if ((i & 3) == 3)
 329                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 330                                         rte_bswap32(reta));
 331         }
 332
 333         /*
 334          * Generate RSS hash based on packet types, TCP/UDP
 335          * port numbers and/or IPv4/v6 src and dst addresses
 336          */
 337         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 338         mrqc = 0;
 339         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 340         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 341         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 342         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 343         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 344         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 345         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 346         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 347         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 348
 349         if (mrqc == 0) {
 350                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 351                         "supported", hf);
 352                 return;
 353         }
 354
 355         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 356 }
 357
 358 static int
 359 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 360 {
 361         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 362         int i, ret;
 363         struct fm10k_tx_queue *txq;
 364         uint64_t base_addr;
 365         uint32_t size;
 366
 367         /* Disable TXINT to avoid possible interrupt */
 368         for (i = 0; i < hw->mac.max_queues; i++)
 369                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 370                                 3 << FM10K_TXINT_TIMER_SHIFT);
 371
 372         /* Setup TX queue */
 373         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 374                 txq = dev->data->tx_queues[i];
 375                 base_addr = txq->hw_ring_phys_addr;
 376                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 377
 378                 /* disable queue to avoid issues while updating state */
 379                 ret = tx_queue_disable(hw, i);
 380                 if (ret) {
 381                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 382                         return -1;
 383                 }
 384
 385                 /* set location and size for descriptor ring */
 386                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 387                                 base_addr & UINT64_LOWER_32BITS_MASK);
 388                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 389                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 390                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 391         }
 392         return 0;
 393 }
 394
 395 static int
 396 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 397 {
 398         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 399         int i, ret;
 400         struct fm10k_rx_queue *rxq;
 401         uint64_t base_addr;
 402         uint32_t size;
 403         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 404         uint16_t buf_size;
 405
 406         /* Disable RXINT to avoid possible interrupt */
 407         for (i = 0; i < hw->mac.max_queues; i++)
 408                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 409                                 3 << FM10K_RXINT_TIMER_SHIFT);
 410
 411         /* Setup RX queues */
 412         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 413                 rxq = dev->data->rx_queues[i];
 414                 base_addr = rxq->hw_ring_phys_addr;
 415                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 416
 417                 /* disable queue to avoid issues while updating state */
 418                 ret = rx_queue_disable(hw, i);
 419                 if (ret) {
 420                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 421                         return -1;
 422                 }
 423
 424                 /* Setup the Base and Length of the Rx Descriptor Ring */
 425                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 426                                 base_addr & UINT64_LOWER_32BITS_MASK);
 427                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 428                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 429                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 430
 431                 /* Configure the Rx buffer size for one buff without split */
 432                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 433                         RTE_PKTMBUF_HEADROOM);
 434                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 435                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 436
 437                 /* It adds dual VLAN length for supporting dual VLAN */
 438                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 439                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size){
 440                         dev->data->scattered_rx = 1;
 441                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 442                 }
 443
 444                 /* Enable drop on empty, it's RO for VF */
 445                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 446                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 447
 448                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 449                 FM10K_WRITE_FLUSH(hw);
 450         }
 451
 452         if (dev->data->dev_conf.rxmode.enable_scatter) {
 453                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 454                 dev->data->scattered_rx = 1;
 455         }
 456
 457         /* Configure RSS if applicable */
 458         fm10k_dev_mq_rx_configure(dev);
 459         return 0;
 460 }
 461
 462 static int
 463 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 464 {
 465         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 466         int err = -1;
 467         uint32_t reg;
 468         struct fm10k_rx_queue *rxq;
 469
 470         PMD_INIT_FUNC_TRACE();
 471
 472         if (rx_queue_id < dev->data->nb_rx_queues) {
 473                 rxq = dev->data->rx_queues[rx_queue_id];
 474                 err = rx_queue_reset(rxq);
 475                 if (err == -ENOMEM) {
 476                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 477                         return err;
 478                 } else if (err == -EINVAL) {
 479                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 480                                 " %d", err);
 481                         return err;
 482                 }
 483
 484                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 485                  * Note: this must be done AFTER the queue is enabled on real
 486                  * hardware, but BEFORE the queue is enabled when using the
 487                  * emulation platform. Do it in both places for now and remove
 488                  * this comment and the following two register writes when the
 489                  * emulation platform is no longer being used.
 490                  */
 491                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 492                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 493
 494                 /* Set PF ownership flag for PF devices */
 495                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 496                 if (hw->mac.type == fm10k_mac_pf)
 497                         reg |= FM10K_RXQCTL_PF;
 498                 reg |= FM10K_RXQCTL_ENABLE;
 499                 /* enable RX queue */
 500                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 501                 FM10K_WRITE_FLUSH(hw);
 502
 503                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 504                  * Note: this must be done AFTER the queue is enabled
 505                  */
 506                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 507                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 508         }
 509
 510         return err;
 511 }
 512
 513 static int
 514 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 515 {
 516         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 517
 518         PMD_INIT_FUNC_TRACE();
 519
 520         if (rx_queue_id < dev->data->nb_rx_queues) {
 521                 /* Disable RX queue */
 522                 rx_queue_disable(hw, rx_queue_id);
 523
 524                 /* Free mbuf and clean HW ring */
 525                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 526         }
 527
 528         return 0;
 529 }
 530
 531 static int
 532 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 533 {
 534         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 535         /** @todo - this should be defined in the shared code */
 536 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 537         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 538         int err = 0;
 539
 540         PMD_INIT_FUNC_TRACE();
 541
 542         if (tx_queue_id < dev->data->nb_tx_queues) {
 543                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 544
 545                 /* reset head and tail pointers */
 546                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 547                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 548
 549                 /* enable TX queue */
 550                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 551                                         FM10K_TXDCTL_ENABLE | txdctl);
 552                 FM10K_WRITE_FLUSH(hw);
 553         } else
 554                 err = -1;
 555
 556         return err;
 557 }
 558
 559 static int
 560 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 561 {
 562         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 563
 564         PMD_INIT_FUNC_TRACE();
 565
 566         if (tx_queue_id < dev->data->nb_tx_queues) {
 567                 tx_queue_disable(hw, tx_queue_id);
 568                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 569         }
 570
 571         return 0;
 572 }
 573
 574 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 575 {
 576         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 577                 != FM10K_DGLORTMAP_NONE);
 578 }
 579
 580 static void
 581 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 582 {
 583         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 584         int status;
 585
 586         PMD_INIT_FUNC_TRACE();
 587
 588         /* Return if it didn't acquire valid glort range */
 589         if (!fm10k_glort_valid(hw))
 590                 return;
 591
 592         fm10k_mbx_lock(hw);
 593         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 594                                 FM10K_XCAST_MODE_PROMISC);
 595         fm10k_mbx_unlock(hw);
 596
 597         if (status != FM10K_SUCCESS)
 598                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 599 }
 600
 601 static void
 602 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 603 {
 604         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 605         uint8_t mode;
 606         int status;
 607
 608         PMD_INIT_FUNC_TRACE();
 609
 610         /* Return if it didn't acquire valid glort range */
 611         if (!fm10k_glort_valid(hw))
 612                 return;
 613
 614         if (dev->data->all_multicast == 1)
 615                 mode = FM10K_XCAST_MODE_ALLMULTI;
 616         else
 617                 mode = FM10K_XCAST_MODE_NONE;
 618
 619         fm10k_mbx_lock(hw);
 620         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 621                                 mode);
 622         fm10k_mbx_unlock(hw);
 623
 624         if (status != FM10K_SUCCESS)
 625                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 626 }
 627
 628 static void
 629 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 630 {
 631         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 632         int status;
 633
 634         PMD_INIT_FUNC_TRACE();
 635
 636         /* Return if it didn't acquire valid glort range */
 637         if (!fm10k_glort_valid(hw))
 638                 return;
 639
 640         /* If promiscuous mode is enabled, it doesn't make sense to enable
 641          * allmulticast and disable promiscuous since fm10k only can select
 642          * one of the modes.
 643          */
 644         if (dev->data->promiscuous) {
 645                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 646                         "needn't enable allmulticast");
 647                 return;
 648         }
 649
 650         fm10k_mbx_lock(hw);
 651         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 652                                 FM10K_XCAST_MODE_ALLMULTI);
 653         fm10k_mbx_unlock(hw);
 654
 655         if (status != FM10K_SUCCESS)
 656                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 657 }
 658
 659 static void
 660 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 661 {
 662         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 663         int status;
 664
 665         PMD_INIT_FUNC_TRACE();
 666
 667         /* Return if it didn't acquire valid glort range */
 668         if (!fm10k_glort_valid(hw))
 669                 return;
 670
 671         if (dev->data->promiscuous) {
 672                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 673                         "since promisc mode is enabled");
 674                 return;
 675         }
 676
 677         fm10k_mbx_lock(hw);
 678         /* Change mode to unicast mode */
 679         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 680                                 FM10K_XCAST_MODE_NONE);
 681         fm10k_mbx_unlock(hw);
 682
 683         if (status != FM10K_SUCCESS)
 684                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 685 }
 686
 687 /* fls = find last set bit = 32 minus the number of leading zeros */
 688 #ifndef fls
 689 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 690 #endif
 691 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 692 static int
 693 fm10k_dev_start(struct rte_eth_dev *dev)
 694 {
 695         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 696         int i, diag;
 697
 698         PMD_INIT_FUNC_TRACE();
 699
 700         /* stop, init, then start the hw */
 701         diag = fm10k_stop_hw(hw);
 702         if (diag != FM10K_SUCCESS) {
 703                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 704                 return -EIO;
 705         }
 706
 707         diag = fm10k_init_hw(hw);
 708         if (diag != FM10K_SUCCESS) {
 709                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 710                 return -EIO;
 711         }
 712
 713         diag = fm10k_start_hw(hw);
 714         if (diag != FM10K_SUCCESS) {
 715                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 716                 return -EIO;
 717         }
 718
 719         diag = fm10k_dev_tx_init(dev);
 720         if (diag) {
 721                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 722                 return diag;
 723         }
 724
 725         diag = fm10k_dev_rx_init(dev);
 726         if (diag) {
 727                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 728                 return diag;
 729         }
 730
 731         if (hw->mac.type == fm10k_mac_pf) {
 732                 /* Establish only VSI 0 as valid */
 733                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 734
 735                 /* Configure RSS bits used in RETA table */
 736                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 737                                 fls(dev->data->nb_rx_queues - 1) <<
 738                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 739
 740                 /* Invalidate all other GLORT entries */
 741                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 742                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 743                                         FM10K_DGLORTMAP_NONE);
 744         }
 745
 746         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 747                 struct fm10k_rx_queue *rxq;
 748                 rxq = dev->data->rx_queues[i];
 749
 750                 if (rxq->rx_deferred_start)
 751                         continue;
 752                 diag = fm10k_dev_rx_queue_start(dev, i);
 753                 if (diag != 0) {
 754                         int j;
 755                         for (j = 0; j < i; ++j)
 756                                 rx_queue_clean(dev->data->rx_queues[j]);
 757                         return diag;
 758                 }
 759         }
 760
 761         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 762                 struct fm10k_tx_queue *txq;
 763                 txq = dev->data->tx_queues[i];
 764
 765                 if (txq->tx_deferred_start)
 766                         continue;
 767                 diag = fm10k_dev_tx_queue_start(dev, i);
 768                 if (diag != 0) {
 769                         int j;
 770                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 771                                 rx_queue_clean(dev->data->rx_queues[j]);
 772                         return diag;
 773                 }
 774         }
 775
 776         return 0;
 777 }
 778
 779 static void
 780 fm10k_dev_stop(struct rte_eth_dev *dev)
 781 {
 782         int i;
 783
 784         PMD_INIT_FUNC_TRACE();
 785
 786         for (i = 0; i < dev->data->nb_tx_queues; i++)
 787                 fm10k_dev_tx_queue_stop(dev, i);
 788
 789         for (i = 0; i < dev->data->nb_rx_queues; i++)
 790                 fm10k_dev_rx_queue_stop(dev, i);
 791 }
 792
 793 static void
 794 fm10k_dev_close(struct rte_eth_dev *dev)
 795 {
 796         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 797
 798         PMD_INIT_FUNC_TRACE();
 799
 800         /* Stop mailbox service first */
 801         fm10k_close_mbx_service(hw);
 802         fm10k_dev_stop(dev);
 803         fm10k_stop_hw(hw);
 804 }
 805
 806 static int
 807 fm10k_link_update(struct rte_eth_dev *dev,
 808         __rte_unused int wait_to_complete)
 809 {
 810         PMD_INIT_FUNC_TRACE();
 811
 812         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 813          * x8 PCIe interface. For now, we leave the speed undefined since there
 814          * is no 50Gbps Ethernet. */
 815         dev->data->dev_link.link_speed  = 0;
 816         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 817         dev->data->dev_link.link_status = 1;
 818
 819         return 0;
 820 }
 821
 822 static void
 823 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 824 {
 825         uint64_t ipackets, opackets, ibytes, obytes;
 826         struct fm10k_hw *hw =
 827                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 828         struct fm10k_hw_stats *hw_stats =
 829                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 830         int i;
 831
 832         PMD_INIT_FUNC_TRACE();
 833
 834         fm10k_update_hw_stats(hw, hw_stats);
 835
 836         ipackets = opackets = ibytes = obytes = 0;
 837         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 838                 (i < FM10K_MAX_QUEUES_PF); ++i) {
 839                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 840                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 841                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 842                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 843                 ipackets += stats->q_ipackets[i];
 844                 opackets += stats->q_opackets[i];
 845                 ibytes   += stats->q_ibytes[i];
 846                 obytes   += stats->q_obytes[i];
 847         }
 848         stats->ipackets = ipackets;
 849         stats->opackets = opackets;
 850         stats->ibytes = ibytes;
 851         stats->obytes = obytes;
 852 }
 853
 854 static void
 855 fm10k_stats_reset(struct rte_eth_dev *dev)
 856 {
 857         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 858         struct fm10k_hw_stats *hw_stats =
 859                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 860
 861         PMD_INIT_FUNC_TRACE();
 862
 863         memset(hw_stats, 0, sizeof(*hw_stats));
 864         fm10k_rebind_hw_stats(hw, hw_stats);
 865 }
 866
 867 static void
 868 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 869         struct rte_eth_dev_info *dev_info)
 870 {
 871         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 872
 873         PMD_INIT_FUNC_TRACE();
 874
 875         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 876         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 877         dev_info->max_rx_queues      = hw->mac.max_queues;
 878         dev_info->max_tx_queues      = hw->mac.max_queues;
 879         dev_info->max_mac_addrs      = 1;
 880         dev_info->max_hash_mac_addrs = 0;
 881         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 882         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 883         dev_info->rx_offload_capa =
 884                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 885                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 886                 DEV_RX_OFFLOAD_TCP_CKSUM;
 887         dev_info->tx_offload_capa    = 0;
 888         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 889
 890         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 891                 .rx_thresh = {
 892                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 893                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 894                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 895                 },
 896                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 897                 .rx_drop_en = 0,
 898         };
 899
 900         dev_info->default_txconf = (struct rte_eth_txconf) {
 901                 .tx_thresh = {
 902                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 903                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 904                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 905                 },
 906                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 907                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 908                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 909                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 910         };
 911
 912 }
 913
 914 static int
 915 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 916 {
 917         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 918
 919         PMD_INIT_FUNC_TRACE();
 920
 921         /* @todo - add support for the VF */
 922         if (hw->mac.type != fm10k_mac_pf)
 923                 return -ENOTSUP;
 924
 925         return fm10k_update_vlan(hw, vlan_id, 0, on);
 926 }
 927
 928 static inline int
 929 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
 930 {
 931         if ((request < min) || (request > max) || ((request % mult) != 0))
 932                 return -1;
 933         else
 934                 return 0;
 935 }
 936
 937 /*
 938  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
 939  * the physical address is required. If the memzone is already created, then
 940  * this function returns a pointer to the existing memzone.
 941  */
 942 static inline const struct rte_memzone *
 943 allocate_hw_ring(const char *driver_name, const char *ring_name,
 944         uint8_t port_id, uint16_t queue_id, int socket_id,
 945         uint32_t size, uint32_t align)
 946 {
 947         char name[RTE_MEMZONE_NAMESIZE];
 948         const struct rte_memzone *mz;
 949
 950         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
 951                  driver_name, ring_name, port_id, queue_id, socket_id);
 952
 953         /* return the memzone if it already exists */
 954         mz = rte_memzone_lookup(name);
 955         if (mz)
 956                 return mz;
 957
 958 #ifdef RTE_LIBRTE_XEN_DOM0
 959         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
 960                                            RTE_PGSIZE_2M);
 961 #else
 962         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
 963 #endif
 964 }
 965
 966 static inline int
 967 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
 968 {
 969         if ((request < min) || (request > max) || ((div % request) != 0))
 970                 return -1;
 971         else
 972                 return 0;
 973 }
 974
 975 static inline int
 976 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
 977 {
 978         uint16_t rx_free_thresh;
 979
 980         if (conf->rx_free_thresh == 0)
 981                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
 982         else
 983                 rx_free_thresh = conf->rx_free_thresh;
 984
 985         /* make sure the requested threshold satisfies the constraints */
 986         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
 987                         FM10K_RX_FREE_THRESH_MAX(q),
 988                         FM10K_RX_FREE_THRESH_DIV(q),
 989                         rx_free_thresh)) {
 990                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
 991                         "less than or equal to %u, "
 992                         "greater than or equal to %u, "
 993                         "and a divisor of %u",
 994                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
 995                         FM10K_RX_FREE_THRESH_MIN(q),
 996                         FM10K_RX_FREE_THRESH_DIV(q));
 997                 return (-EINVAL);
 998         }
 999
1000         q->alloc_thresh = rx_free_thresh;
1001         q->drop_en = conf->rx_drop_en;
1002         q->rx_deferred_start = conf->rx_deferred_start;
1003
1004         return 0;
1005 }
1006
1007 /*
1008  * Hardware requires specific alignment for Rx packet buffers. At
1009  * least one of the following two conditions must be satisfied.
1010  *  1. Address is 512B aligned
1011  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1012  *
1013  * As such, the driver may need to adjust the DMA address within the
1014  * buffer by up to 512B. The mempool element size is checked here
1015  * to make sure a maximally sized Ethernet frame can still be wholly
1016  * contained within the buffer after 512B alignment.
1017  *
1018  * return 1 if the element size is valid, otherwise return 0.
1019  */
1020 static int
1021 mempool_element_size_valid(struct rte_mempool *mp)
1022 {
1023         uint32_t min_size;
1024
1025         /* elt_size includes mbuf header and headroom */
1026         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1027                         RTE_PKTMBUF_HEADROOM;
1028
1029         /* account for up to 512B of alignment */
1030         min_size -= FM10K_RX_BUFF_ALIGN;
1031
1032         /* sanity check for overflow */
1033         if (min_size > mp->elt_size)
1034                 return 0;
1035
1036         if (min_size < ETHER_MAX_VLAN_FRAME_LEN)
1037                 return 0;
1038
1039         /* size is valid */
1040         return 1;
1041 }
1042
1043 static int
1044 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1045         uint16_t nb_desc, unsigned int socket_id,
1046         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1047 {
1048         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1049         struct fm10k_rx_queue *q;
1050         const struct rte_memzone *mz;
1051
1052         PMD_INIT_FUNC_TRACE();
1053
1054         /* make sure the mempool element size can account for alignment. */
1055         if (!mempool_element_size_valid(mp)) {
1056                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1057                 return (-EINVAL);
1058         }
1059
1060         /* make sure a valid number of descriptors have been requested */
1061         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1062                                 FM10K_MULT_RX_DESC, nb_desc)) {
1063                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1064                         "less than or equal to %"PRIu32", "
1065                         "greater than or equal to %u, "
1066                         "and a multiple of %u",
1067                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1068                         FM10K_MULT_RX_DESC);
1069                 return (-EINVAL);
1070         }
1071
1072         /*
1073          * if this queue existed already, free the associated memory. The
1074          * queue cannot be reused in case we need to allocate memory on
1075          * different socket than was previously used.
1076          */
1077         if (dev->data->rx_queues[queue_id] != NULL) {
1078                 rx_queue_free(dev->data->rx_queues[queue_id]);
1079                 dev->data->rx_queues[queue_id] = NULL;
1080         }
1081
1082         /* allocate memory for the queue structure */
1083         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1084                                 socket_id);
1085         if (q == NULL) {
1086                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1087                 return (-ENOMEM);
1088         }
1089
1090         /* setup queue */
1091         q->mp = mp;
1092         q->nb_desc = nb_desc;
1093         q->port_id = dev->data->port_id;
1094         q->queue_id = queue_id;
1095         q->tail_ptr = (volatile uint32_t *)
1096                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1097         if (handle_rxconf(q, conf))
1098                 return (-EINVAL);
1099
1100         /* allocate memory for the software ring */
1101         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1102                                         nb_desc * sizeof(struct rte_mbuf *),
1103                                         RTE_CACHE_LINE_SIZE, socket_id);
1104         if (q->sw_ring == NULL) {
1105                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1106                 rte_free(q);
1107                 return (-ENOMEM);
1108         }
1109
1110         /*
1111          * allocate memory for the hardware descriptor ring. A memzone large
1112          * enough to hold the maximum ring size is requested to allow for
1113          * resizing in later calls to the queue setup function.
1114          */
1115         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1116                                 dev->data->port_id, queue_id, socket_id,
1117                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1118         if (mz == NULL) {
1119                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1120                 rte_free(q->sw_ring);
1121                 rte_free(q);
1122                 return (-ENOMEM);
1123         }
1124         q->hw_ring = mz->addr;
1125         q->hw_ring_phys_addr = mz->phys_addr;
1126
1127         dev->data->rx_queues[queue_id] = q;
1128         return 0;
1129 }
1130
1131 static void
1132 fm10k_rx_queue_release(void *queue)
1133 {
1134         PMD_INIT_FUNC_TRACE();
1135
1136         rx_queue_free(queue);
1137 }
1138
1139 static inline int
1140 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1141 {
1142         uint16_t tx_free_thresh;
1143         uint16_t tx_rs_thresh;
1144
1145         /* constraint MACROs require that tx_free_thresh is configured
1146          * before tx_rs_thresh */
1147         if (conf->tx_free_thresh == 0)
1148                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1149         else
1150                 tx_free_thresh = conf->tx_free_thresh;
1151
1152         /* make sure the requested threshold satisfies the constraints */
1153         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1154                         FM10K_TX_FREE_THRESH_MAX(q),
1155                         FM10K_TX_FREE_THRESH_DIV(q),
1156                         tx_free_thresh)) {
1157                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1158                         "less than or equal to %u, "
1159                         "greater than or equal to %u, "
1160                         "and a divisor of %u",
1161                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1162                         FM10K_TX_FREE_THRESH_MIN(q),
1163                         FM10K_TX_FREE_THRESH_DIV(q));
1164                 return (-EINVAL);
1165         }
1166
1167         q->free_thresh = tx_free_thresh;
1168
1169         if (conf->tx_rs_thresh == 0)
1170                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1171         else
1172                 tx_rs_thresh = conf->tx_rs_thresh;
1173
1174         q->tx_deferred_start = conf->tx_deferred_start;
1175
1176         /* make sure the requested threshold satisfies the constraints */
1177         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1178                         FM10K_TX_RS_THRESH_MAX(q),
1179                         FM10K_TX_RS_THRESH_DIV(q),
1180                         tx_rs_thresh)) {
1181                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1182                         "less than or equal to %u, "
1183                         "greater than or equal to %u, "
1184                         "and a divisor of %u",
1185                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1186                         FM10K_TX_RS_THRESH_MIN(q),
1187                         FM10K_TX_RS_THRESH_DIV(q));
1188                 return (-EINVAL);
1189         }
1190
1191         q->rs_thresh = tx_rs_thresh;
1192
1193         return 0;
1194 }
1195
1196 static int
1197 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1198         uint16_t nb_desc, unsigned int socket_id,
1199         const struct rte_eth_txconf *conf)
1200 {
1201         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1202         struct fm10k_tx_queue *q;
1203         const struct rte_memzone *mz;
1204
1205         PMD_INIT_FUNC_TRACE();
1206
1207         /* make sure a valid number of descriptors have been requested */
1208         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1209                                 FM10K_MULT_TX_DESC, nb_desc)) {
1210                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1211                         "less than or equal to %"PRIu32", "
1212                         "greater than or equal to %u, "
1213                         "and a multiple of %u",
1214                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1215                         FM10K_MULT_TX_DESC);
1216                 return (-EINVAL);
1217         }
1218
1219         /*
1220          * if this queue existed already, free the associated memory. The
1221          * queue cannot be reused in case we need to allocate memory on
1222          * different socket than was previously used.
1223          */
1224         if (dev->data->tx_queues[queue_id] != NULL) {
1225                 tx_queue_free(dev->data->tx_queues[queue_id]);
1226                 dev->data->tx_queues[queue_id] = NULL;
1227         }
1228
1229         /* allocate memory for the queue structure */
1230         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1231                                 socket_id);
1232         if (q == NULL) {
1233                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1234                 return (-ENOMEM);
1235         }
1236
1237         /* setup queue */
1238         q->nb_desc = nb_desc;
1239         q->port_id = dev->data->port_id;
1240         q->queue_id = queue_id;
1241         q->tail_ptr = (volatile uint32_t *)
1242                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1243         if (handle_txconf(q, conf))
1244                 return (-EINVAL);
1245
1246         /* allocate memory for the software ring */
1247         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1248                                         nb_desc * sizeof(struct rte_mbuf *),
1249                                         RTE_CACHE_LINE_SIZE, socket_id);
1250         if (q->sw_ring == NULL) {
1251                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1252                 rte_free(q);
1253                 return (-ENOMEM);
1254         }
1255
1256         /*
1257          * allocate memory for the hardware descriptor ring. A memzone large
1258          * enough to hold the maximum ring size is requested to allow for
1259          * resizing in later calls to the queue setup function.
1260          */
1261         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1262                                 dev->data->port_id, queue_id, socket_id,
1263                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1264         if (mz == NULL) {
1265                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1266                 rte_free(q->sw_ring);
1267                 rte_free(q);
1268                 return (-ENOMEM);
1269         }
1270         q->hw_ring = mz->addr;
1271         q->hw_ring_phys_addr = mz->phys_addr;
1272
1273         /*
1274          * allocate memory for the RS bit tracker. Enough slots to hold the
1275          * descriptor index for each RS bit needing to be set are required.
1276          */
1277         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1278                                 ((nb_desc + 1) / q->rs_thresh) *
1279                                 sizeof(uint16_t),
1280                                 RTE_CACHE_LINE_SIZE, socket_id);
1281         if (q->rs_tracker.list == NULL) {
1282                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1283                 rte_free(q->sw_ring);
1284                 rte_free(q);
1285                 return (-ENOMEM);
1286         }
1287
1288         dev->data->tx_queues[queue_id] = q;
1289         return 0;
1290 }
1291
1292 static void
1293 fm10k_tx_queue_release(void *queue)
1294 {
1295         PMD_INIT_FUNC_TRACE();
1296
1297         tx_queue_free(queue);
1298 }
1299
1300 static int
1301 fm10k_reta_update(struct rte_eth_dev *dev,
1302                         struct rte_eth_rss_reta_entry64 *reta_conf,
1303                         uint16_t reta_size)
1304 {
1305         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1306         uint16_t i, j, idx, shift;
1307         uint8_t mask;
1308         uint32_t reta;
1309
1310         PMD_INIT_FUNC_TRACE();
1311
1312         if (reta_size > FM10K_MAX_RSS_INDICES) {
1313                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1314                         "(%d) doesn't match the number hardware can supported "
1315                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1316                 return -EINVAL;
1317         }
1318
1319         /*
1320          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1321          * 128-entries in 32 registers
1322          */
1323         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1324                 idx = i / RTE_RETA_GROUP_SIZE;
1325                 shift = i % RTE_RETA_GROUP_SIZE;
1326                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1327                                 BIT_MASK_PER_UINT32);
1328                 if (mask == 0)
1329                         continue;
1330
1331                 reta = 0;
1332                 if (mask != BIT_MASK_PER_UINT32)
1333                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1334
1335                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1336                         if (mask & (0x1 << j)) {
1337                                 if (mask != 0xF)
1338                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1339                                 reta |= reta_conf[idx].reta[shift + j] <<
1340                                                 (CHAR_BIT * j);
1341                         }
1342                 }
1343                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1344         }
1345
1346         return 0;
1347 }
1348
1349 static int
1350 fm10k_reta_query(struct rte_eth_dev *dev,
1351                         struct rte_eth_rss_reta_entry64 *reta_conf,
1352                         uint16_t reta_size)
1353 {
1354         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1355         uint16_t i, j, idx, shift;
1356         uint8_t mask;
1357         uint32_t reta;
1358
1359         PMD_INIT_FUNC_TRACE();
1360
1361         if (reta_size < FM10K_MAX_RSS_INDICES) {
1362                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1363                         "(%d) doesn't match the number hardware can supported "
1364                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1365                 return -EINVAL;
1366         }
1367
1368         /*
1369          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1370          * 128-entries in 32 registers
1371          */
1372         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1373                 idx = i / RTE_RETA_GROUP_SIZE;
1374                 shift = i % RTE_RETA_GROUP_SIZE;
1375                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1376                                 BIT_MASK_PER_UINT32);
1377                 if (mask == 0)
1378                         continue;
1379
1380                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1381                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1382                         if (mask & (0x1 << j))
1383                                 reta_conf[idx].reta[shift + j] = ((reta >>
1384                                         CHAR_BIT * j) & UINT8_MAX);
1385                 }
1386         }
1387
1388         return 0;
1389 }
1390
1391 static int
1392 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1393         struct rte_eth_rss_conf *rss_conf)
1394 {
1395         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1396         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1397         uint32_t mrqc;
1398         uint64_t hf = rss_conf->rss_hf;
1399         int i;
1400
1401         PMD_INIT_FUNC_TRACE();
1402
1403         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1404                 FM10K_RSSRK_ENTRIES_PER_REG)
1405                 return -EINVAL;
1406
1407         if (hf == 0)
1408                 return -EINVAL;
1409
1410         mrqc = 0;
1411         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1412         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1413         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1414         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1415         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1416         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1417         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1418         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1419         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1420
1421         /* If the mapping doesn't fit any supported, return */
1422         if (mrqc == 0)
1423                 return -EINVAL;
1424
1425         if (key != NULL)
1426                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1427                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1428
1429         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1430
1431         return 0;
1432 }
1433
1434 static int
1435 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1436         struct rte_eth_rss_conf *rss_conf)
1437 {
1438         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1439         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1440         uint32_t mrqc;
1441         uint64_t hf;
1442         int i;
1443
1444         PMD_INIT_FUNC_TRACE();
1445
1446         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1447                                 FM10K_RSSRK_ENTRIES_PER_REG)
1448                 return -EINVAL;
1449
1450         if (key != NULL)
1451                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1452                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1453
1454         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1455         hf = 0;
1456         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1457         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1458         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1459         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1460         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1461         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1462         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1463         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1464         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1465
1466         rss_conf->rss_hf = hf;
1467
1468         return 0;
1469 }
1470
1471 static void
1472 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1473 {
1474         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1475         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1476
1477         /* Bind all local non-queue interrupt to vector 0 */
1478         int_map |= 0;
1479
1480         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1481         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1482         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1483         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1484         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1485         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1486
1487         /* Enable misc causes */
1488         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1489                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1490                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1491                                 FM10K_EIMR_ENABLE(MAILBOX) |
1492                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1493                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1494                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1495                                 FM10K_EIMR_ENABLE(VFLR));
1496
1497         /* Enable ITR 0 */
1498         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1499                                         FM10K_ITR_MASK_CLEAR);
1500         FM10K_WRITE_FLUSH(hw);
1501 }
1502
1503 static void
1504 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1505 {
1506         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1507         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1508
1509         /* Bind all local non-queue interrupt to vector 0 */
1510         int_map |= 0;
1511
1512         /* Only INT 0 available, other 15 are reserved. */
1513         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1514
1515         /* Enable ITR 0 */
1516         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1517                                         FM10K_ITR_MASK_CLEAR);
1518         FM10K_WRITE_FLUSH(hw);
1519 }
1520
1521 static int
1522 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1523 {
1524         struct fm10k_fault fault;
1525         int err;
1526         const char *estr = "Unknown error";
1527
1528         /* Process PCA fault */
1529         if (eicr & FM10K_EIMR_PCA_FAULT) {
1530                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1531                 if (err)
1532                         goto error;
1533                 switch (fault.type) {
1534                 case PCA_NO_FAULT:
1535                         estr = "PCA_NO_FAULT"; break;
1536                 case PCA_UNMAPPED_ADDR:
1537                         estr = "PCA_UNMAPPED_ADDR"; break;
1538                 case PCA_BAD_QACCESS_PF:
1539                         estr = "PCA_BAD_QACCESS_PF"; break;
1540                 case PCA_BAD_QACCESS_VF:
1541                         estr = "PCA_BAD_QACCESS_VF"; break;
1542                 case PCA_MALICIOUS_REQ:
1543                         estr = "PCA_MALICIOUS_REQ"; break;
1544                 case PCA_POISONED_TLP:
1545                         estr = "PCA_POISONED_TLP"; break;
1546                 case PCA_TLP_ABORT:
1547                         estr = "PCA_TLP_ABORT"; break;
1548                 default:
1549                         goto error;
1550                 }
1551                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1552                         estr, fault.func ? "VF" : "PF", fault.func,
1553                         fault.address, fault.specinfo);
1554         }
1555
1556         /* Process THI fault */
1557         if (eicr & FM10K_EIMR_THI_FAULT) {
1558                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1559                 if (err)
1560                         goto error;
1561                 switch (fault.type) {
1562                 case THI_NO_FAULT:
1563                         estr = "THI_NO_FAULT"; break;
1564                 case THI_MAL_DIS_Q_FAULT:
1565                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1566                 default:
1567                         goto error;
1568                 }
1569                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1570                         estr, fault.func ? "VF" : "PF", fault.func,
1571                         fault.address, fault.specinfo);
1572         }
1573
1574         /* Process FUM fault */
1575         if (eicr & FM10K_EIMR_FUM_FAULT) {
1576                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1577                 if (err)
1578                         goto error;
1579                 switch (fault.type) {
1580                 case FUM_NO_FAULT:
1581                         estr = "FUM_NO_FAULT"; break;
1582                 case FUM_UNMAPPED_ADDR:
1583                         estr = "FUM_UNMAPPED_ADDR"; break;
1584                 case FUM_POISONED_TLP:
1585                         estr = "FUM_POISONED_TLP"; break;
1586                 case FUM_BAD_VF_QACCESS:
1587                         estr = "FUM_BAD_VF_QACCESS"; break;
1588                 case FUM_ADD_DECODE_ERR:
1589                         estr = "FUM_ADD_DECODE_ERR"; break;
1590                 case FUM_RO_ERROR:
1591                         estr = "FUM_RO_ERROR"; break;
1592                 case FUM_QPRC_CRC_ERROR:
1593                         estr = "FUM_QPRC_CRC_ERROR"; break;
1594                 case FUM_CSR_TIMEOUT:
1595                         estr = "FUM_CSR_TIMEOUT"; break;
1596                 case FUM_INVALID_TYPE:
1597                         estr = "FUM_INVALID_TYPE"; break;
1598                 case FUM_INVALID_LENGTH:
1599                         estr = "FUM_INVALID_LENGTH"; break;
1600                 case FUM_INVALID_BE:
1601                         estr = "FUM_INVALID_BE"; break;
1602                 case FUM_INVALID_ALIGN:
1603                         estr = "FUM_INVALID_ALIGN"; break;
1604                 default:
1605                         goto error;
1606                 }
1607                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1608                         estr, fault.func ? "VF" : "PF", fault.func,
1609                         fault.address, fault.specinfo);
1610         }
1611
1612         if (estr)
1613                 return 0;
1614         return 0;
1615 error:
1616         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1617         return err;
1618 }
1619
1620 /**
1621  * PF interrupt handler triggered by NIC for handling specific interrupt.
1622  *
1623  * @param handle
1624  *  Pointer to interrupt handle.
1625  * @param param
1626  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1627  *
1628  * @return
1629  *  void
1630  */
1631 static void
1632 fm10k_dev_interrupt_handler_pf(
1633                         __rte_unused struct rte_intr_handle *handle,
1634                         void *param)
1635 {
1636         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1637         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1638         uint32_t cause, status;
1639
1640         if (hw->mac.type != fm10k_mac_pf)
1641                 return;
1642
1643         cause = FM10K_READ_REG(hw, FM10K_EICR);
1644
1645         /* Handle PCI fault cases */
1646         if (cause & FM10K_EICR_FAULT_MASK) {
1647                 PMD_INIT_LOG(ERR, "INT: find fault!");
1648                 fm10k_dev_handle_fault(hw, cause);
1649         }
1650
1651         /* Handle switch up/down */
1652         if (cause & FM10K_EICR_SWITCHNOTREADY)
1653                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1654
1655         if (cause & FM10K_EICR_SWITCHREADY)
1656                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1657
1658         /* Handle mailbox message */
1659         fm10k_mbx_lock(hw);
1660         hw->mbx.ops.process(hw, &hw->mbx);
1661         fm10k_mbx_unlock(hw);
1662
1663         /* Handle SRAM error */
1664         if (cause & FM10K_EICR_SRAMERROR) {
1665                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1666
1667                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1668                 /* Write to clear pending bits */
1669                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1670
1671                 /* Todo: print out error message after shared code  updates */
1672         }
1673
1674         /* Clear these 3 events if having any */
1675         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1676                  FM10K_EICR_SWITCHREADY;
1677         if (cause)
1678                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1679
1680         /* Re-enable interrupt from device side */
1681         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1682                                         FM10K_ITR_MASK_CLEAR);
1683         /* Re-enable interrupt from host side */
1684         rte_intr_enable(&(dev->pci_dev->intr_handle));
1685 }
1686
1687 /**
1688  * VF interrupt handler triggered by NIC for handling specific interrupt.
1689  *
1690  * @param handle
1691  *  Pointer to interrupt handle.
1692  * @param param
1693  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1694  *
1695  * @return
1696  *  void
1697  */
1698 static void
1699 fm10k_dev_interrupt_handler_vf(
1700                         __rte_unused struct rte_intr_handle *handle,
1701                         void *param)
1702 {
1703         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1704         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1705
1706         if (hw->mac.type != fm10k_mac_vf)
1707                 return;
1708
1709         /* Handle mailbox message if lock is acquired */
1710         fm10k_mbx_lock(hw);
1711         hw->mbx.ops.process(hw, &hw->mbx);
1712         fm10k_mbx_unlock(hw);
1713
1714         /* Re-enable interrupt from device side */
1715         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1716                                         FM10K_ITR_MASK_CLEAR);
1717         /* Re-enable interrupt from host side */
1718         rte_intr_enable(&(dev->pci_dev->intr_handle));
1719 }
1720
1721 /* Mailbox message handler in VF */
1722 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1723         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1724         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1725         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1726         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1727 };
1728
1729 /* Mailbox message handler in PF */
1730 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1731         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1732         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1733         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1734         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1735         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1736         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1737         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1738 };
1739
1740 static int
1741 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1742 {
1743         int err;
1744
1745         /* Initialize mailbox lock */
1746         fm10k_mbx_initlock(hw);
1747
1748         /* Replace default message handler with new ones */
1749         if (hw->mac.type == fm10k_mac_pf)
1750                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1751         else
1752                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1753
1754         if (err) {
1755                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1756                                 err);
1757                 return err;
1758         }
1759         /* Connect to SM for PF device or PF for VF device */
1760         return hw->mbx.ops.connect(hw, &hw->mbx);
1761 }
1762
1763 static void
1764 fm10k_close_mbx_service(struct fm10k_hw *hw)
1765 {
1766         /* Disconnect from SM for PF device or PF for VF device */
1767         hw->mbx.ops.disconnect(hw, &hw->mbx);
1768 }
1769
1770 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1771         .dev_configure          = fm10k_dev_configure,
1772         .dev_start              = fm10k_dev_start,
1773         .dev_stop               = fm10k_dev_stop,
1774         .dev_close              = fm10k_dev_close,
1775         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
1776         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
1777         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
1778         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
1779         .stats_get              = fm10k_stats_get,
1780         .stats_reset            = fm10k_stats_reset,
1781         .link_update            = fm10k_link_update,
1782         .dev_infos_get          = fm10k_dev_infos_get,
1783         .vlan_filter_set        = fm10k_vlan_filter_set,
1784         .rx_queue_start         = fm10k_dev_rx_queue_start,
1785         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1786         .tx_queue_start         = fm10k_dev_tx_queue_start,
1787         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1788         .rx_queue_setup         = fm10k_rx_queue_setup,
1789         .rx_queue_release       = fm10k_rx_queue_release,
1790         .tx_queue_setup         = fm10k_tx_queue_setup,
1791         .tx_queue_release       = fm10k_tx_queue_release,
1792         .reta_update            = fm10k_reta_update,
1793         .reta_query             = fm10k_reta_query,
1794         .rss_hash_update        = fm10k_rss_hash_update,
1795         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1796 };
1797
1798 static int
1799 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1800 {
1801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1802         int diag;
1803
1804         PMD_INIT_FUNC_TRACE();
1805
1806         dev->dev_ops = &fm10k_eth_dev_ops;
1807         dev->rx_pkt_burst = &fm10k_recv_pkts;
1808         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1809
1810         if (dev->data->scattered_rx)
1811                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1812
1813         /* only initialize in the primary process */
1814         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1815                 return 0;
1816
1817         /* Vendor and Device ID need to be set before init of shared code */
1818         memset(hw, 0, sizeof(*hw));
1819         hw->device_id = dev->pci_dev->id.device_id;
1820         hw->vendor_id = dev->pci_dev->id.vendor_id;
1821         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1822         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1823         hw->revision_id = 0;
1824         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1825         if (hw->hw_addr == NULL) {
1826                 PMD_INIT_LOG(ERR, "Bad mem resource."
1827                         " Try to blacklist unused devices.");
1828                 return -EIO;
1829         }
1830
1831         /* Store fm10k_adapter pointer */
1832         hw->back = dev->data->dev_private;
1833
1834         /* Initialize the shared code */
1835         diag = fm10k_init_shared_code(hw);
1836         if (diag != FM10K_SUCCESS) {
1837                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1838                 return -EIO;
1839         }
1840
1841         /*
1842          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1843          * there is no way to get link status without reading BAR4.  Until this
1844          * works, assume we have maximum bandwidth.
1845          * @todo - fix bus info
1846          */
1847         hw->bus_caps.speed = fm10k_bus_speed_8000;
1848         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1849         hw->bus_caps.payload = fm10k_bus_payload_512;
1850         hw->bus.speed = fm10k_bus_speed_8000;
1851         hw->bus.width = fm10k_bus_width_pcie_x8;
1852         hw->bus.payload = fm10k_bus_payload_256;
1853
1854         /* Initialize the hw */
1855         diag = fm10k_init_hw(hw);
1856         if (diag != FM10K_SUCCESS) {
1857                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1858                 return -EIO;
1859         }
1860
1861         /* Initialize MAC address(es) */
1862         dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1863         if (dev->data->mac_addrs == NULL) {
1864                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1865                 return -ENOMEM;
1866         }
1867
1868         diag = fm10k_read_mac_addr(hw);
1869         if (diag != FM10K_SUCCESS) {
1870                 /*
1871                  * TODO: remove special handling on VF. Need shared code to
1872                  * fix first.
1873                  */
1874                 if (hw->mac.type == fm10k_mac_pf) {
1875                         PMD_INIT_LOG(ERR, "Read MAC addr failed: %d", diag);
1876                         return -EIO;
1877                 } else {
1878                         /* Generate a random addr */
1879                         eth_random_addr(hw->mac.addr);
1880                         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1881                 }
1882         }
1883
1884         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1885                         &dev->data->mac_addrs[0]);
1886
1887         /* Reset the hw statistics */
1888         fm10k_stats_reset(dev);
1889
1890         /* Reset the hw */
1891         diag = fm10k_reset_hw(hw);
1892         if (diag != FM10K_SUCCESS) {
1893                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1894                 return -EIO;
1895         }
1896
1897         /* Setup mailbox service */
1898         diag = fm10k_setup_mbx_service(hw);
1899         if (diag != FM10K_SUCCESS) {
1900                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1901                 return -EIO;
1902         }
1903
1904         /*PF/VF has different interrupt handling mechanism */
1905         if (hw->mac.type == fm10k_mac_pf) {
1906                 /* register callback func to eal lib */
1907                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1908                         fm10k_dev_interrupt_handler_pf, (void *)dev);
1909
1910                 /* enable MISC interrupt */
1911                 fm10k_dev_enable_intr_pf(dev);
1912         } else { /* VF */
1913                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1914                         fm10k_dev_interrupt_handler_vf, (void *)dev);
1915
1916                 fm10k_dev_enable_intr_vf(dev);
1917         }
1918
1919         /*
1920          * Below function will trigger operations on mailbox, acquire lock to
1921          * avoid race condition from interrupt handler. Operations on mailbox
1922          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
1923          * will handle and generate an interrupt to our side. Then,  FIFO in
1924          * mailbox will be touched.
1925          */
1926         fm10k_mbx_lock(hw);
1927         /* Enable port first */
1928         hw->mac.ops.update_lport_state(hw, 0, 0, 1);
1929
1930         /* Update default vlan */
1931         hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
1932
1933         /*
1934          * Add default mac/vlan filter. glort is assigned by SM for PF, while is
1935          * unused for VF. PF will assign correct glort for VF.
1936          */
1937         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
1938                               hw->mac.default_vid, 1, 0);
1939
1940         /* Set unicast mode by default. App can change to other mode in other
1941          * API func.
1942          */
1943         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1944                                         FM10K_XCAST_MODE_NONE);
1945
1946         fm10k_mbx_unlock(hw);
1947
1948         /* enable uio intr after callback registered */
1949         rte_intr_enable(&(dev->pci_dev->intr_handle));
1950
1951         return 0;
1952 }
1953
1954 /*
1955  * The set of PCI devices this driver supports. This driver will enable both PF
1956  * and SRIOV-VF devices.
1957  */
1958 static const struct rte_pci_id pci_id_fm10k_map[] = {
1959 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1960 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1961 #include "rte_pci_dev_ids.h"
1962         { .vendor_id = 0, /* sentinel */ },
1963 };
1964
1965 static struct eth_driver rte_pmd_fm10k = {
1966         .pci_drv = {
1967                 .name = "rte_pmd_fm10k",
1968                 .id_table = pci_id_fm10k_map,
1969                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1970         },
1971         .eth_dev_init = eth_fm10k_dev_init,
1972         .dev_private_size = sizeof(struct fm10k_adapter),
1973 };
1974
1975 /*
1976  * Driver initialization routine.
1977  * Invoked once at EAL init time.
1978  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
1979  */
1980 static int
1981 rte_pmd_fm10k_init(__rte_unused const char *name,
1982         __rte_unused const char *params)
1983 {
1984         PMD_INIT_FUNC_TRACE();
1985         rte_eth_driver_register(&rte_pmd_fm10k);
1986         return 0;
1987 }
1988
1989 static struct rte_driver rte_fm10k_driver = {
1990         .type = PMD_PDEV,
1991         .init = rte_pmd_fm10k_init,
1992 };
1993
1994 PMD_REGISTER_DRIVER(rte_fm10k_driver);