drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62
  63 static void
  64 fm10k_mbx_initlock(struct fm10k_hw *hw)
  65 {
  66         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  67 }
  68
  69 static void
  70 fm10k_mbx_lock(struct fm10k_hw *hw)
  71 {
  72         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  73                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  74 }
  75
  76 static void
  77 fm10k_mbx_unlock(struct fm10k_hw *hw)
  78 {
  79         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  80 }
  81
  82 /*
  83  * reset queue to initial state, allocate software buffers used when starting
  84  * device.
  85  * return 0 on success
  86  * return -ENOMEM if buffers cannot be allocated
  87  * return -EINVAL if buffers do not satisfy alignment condition
  88  */
  89 static inline int
  90 rx_queue_reset(struct fm10k_rx_queue *q)
  91 {
  92         uint64_t dma_addr;
  93         int i, diag;
  94         PMD_INIT_FUNC_TRACE();
  95
  96         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
  97         if (diag != 0)
  98                 return -ENOMEM;
  99
 100         for (i = 0; i < q->nb_desc; ++i) {
 101                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 102                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 103                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 104                                                 q->nb_desc);
 105                         return -EINVAL;
 106                 }
 107                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 108                 q->hw_ring[i].q.pkt_addr = dma_addr;
 109                 q->hw_ring[i].q.hdr_addr = dma_addr;
 110         }
 111
 112         q->next_dd = 0;
 113         q->next_alloc = 0;
 114         q->next_trigger = q->alloc_thresh - 1;
 115         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 116         return 0;
 117 }
 118
 119 /*
 120  * clean queue, descriptor rings, free software buffers used when stopping
 121  * device.
 122  */
 123 static inline void
 124 rx_queue_clean(struct fm10k_rx_queue *q)
 125 {
 126         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 127         uint32_t i;
 128         PMD_INIT_FUNC_TRACE();
 129
 130         /* zero descriptor rings */
 131         for (i = 0; i < q->nb_desc; ++i)
 132                 q->hw_ring[i] = zero;
 133
 134         /* free software buffers */
 135         for (i = 0; i < q->nb_desc; ++i) {
 136                 if (q->sw_ring[i]) {
 137                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 138                         q->sw_ring[i] = NULL;
 139                 }
 140         }
 141 }
 142
 143 /*
 144  * free all queue memory used when releasing the queue (i.e. configure)
 145  */
 146 static inline void
 147 rx_queue_free(struct fm10k_rx_queue *q)
 148 {
 149         PMD_INIT_FUNC_TRACE();
 150         if (q) {
 151                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 152                 rx_queue_clean(q);
 153                 if (q->sw_ring) {
 154                         rte_free(q->sw_ring);
 155                         q->sw_ring = NULL;
 156                 }
 157                 rte_free(q);
 158                 q = NULL;
 159         }
 160 }
 161
 162 /*
 163  * disable RX queue, wait unitl HW finished necessary flush operation
 164  */
 165 static inline int
 166 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 167 {
 168         uint32_t reg, i;
 169
 170         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 171         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 172                         reg & ~FM10K_RXQCTL_ENABLE);
 173
 174         /* Wait 100us at most */
 175         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 176                 rte_delay_us(1);
 177                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 178                 if (!(reg & FM10K_RXQCTL_ENABLE))
 179                         break;
 180         }
 181
 182         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 183                 return -1;
 184
 185         return 0;
 186 }
 187
 188 /*
 189  * reset queue to initial state, allocate software buffers used when starting
 190  * device
 191  */
 192 static inline void
 193 tx_queue_reset(struct fm10k_tx_queue *q)
 194 {
 195         PMD_INIT_FUNC_TRACE();
 196         q->last_free = 0;
 197         q->next_free = 0;
 198         q->nb_used = 0;
 199         q->nb_free = q->nb_desc - 1;
 200         q->free_trigger = q->nb_free - q->free_thresh;
 201         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 202         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 203 }
 204
 205 /*
 206  * clean queue, descriptor rings, free software buffers used when stopping
 207  * device
 208  */
 209 static inline void
 210 tx_queue_clean(struct fm10k_tx_queue *q)
 211 {
 212         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 213         uint32_t i;
 214         PMD_INIT_FUNC_TRACE();
 215
 216         /* zero descriptor rings */
 217         for (i = 0; i < q->nb_desc; ++i)
 218                 q->hw_ring[i] = zero;
 219
 220         /* free software buffers */
 221         for (i = 0; i < q->nb_desc; ++i) {
 222                 if (q->sw_ring[i]) {
 223                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 224                         q->sw_ring[i] = NULL;
 225                 }
 226         }
 227 }
 228
 229 /*
 230  * free all queue memory used when releasing the queue (i.e. configure)
 231  */
 232 static inline void
 233 tx_queue_free(struct fm10k_tx_queue *q)
 234 {
 235         PMD_INIT_FUNC_TRACE();
 236         if (q) {
 237                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 238                 tx_queue_clean(q);
 239                 if (q->rs_tracker.list) {
 240                         rte_free(q->rs_tracker.list);
 241                         q->rs_tracker.list = NULL;
 242                 }
 243                 if (q->sw_ring) {
 244                         rte_free(q->sw_ring);
 245                         q->sw_ring = NULL;
 246                 }
 247                 rte_free(q);
 248                 q = NULL;
 249         }
 250 }
 251
 252 /*
 253  * disable TX queue, wait unitl HW finished necessary flush operation
 254  */
 255 static inline int
 256 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 257 {
 258         uint32_t reg, i;
 259
 260         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 261         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 262                         reg & ~FM10K_TXDCTL_ENABLE);
 263
 264         /* Wait 100us at most */
 265         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 266                 rte_delay_us(1);
 267                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 268                 if (!(reg & FM10K_TXDCTL_ENABLE))
 269                         break;
 270         }
 271
 272         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 273                 return -1;
 274
 275         return 0;
 276 }
 277
 278 static int
 279 fm10k_dev_configure(struct rte_eth_dev *dev)
 280 {
 281         PMD_INIT_FUNC_TRACE();
 282
 283         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 284                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 285
 286         return 0;
 287 }
 288
 289 static void
 290 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 291 {
 292         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 293         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 294         uint32_t mrqc, *key, i, reta, j;
 295         uint64_t hf;
 296
 297 #define RSS_KEY_SIZE 40
 298         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 299                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 300                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 301                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 302                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 303                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 304         };
 305
 306         if (dev->data->nb_rx_queues == 1 ||
 307             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 308             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 309                 return;
 310
 311         /* random key is rss_intel_key (default) or user provided (rss_key) */
 312         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 313                 key = (uint32_t *)rss_intel_key;
 314         else
 315                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 316
 317         /* Now fill our hash function seeds, 4 bytes at a time */
 318         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 319                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 320
 321         /*
 322          * Fill in redirection table
 323          * The byte-swap is needed because NIC registers are in
 324          * little-endian order.
 325          */
 326         reta = 0;
 327         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 328                 if (j == dev->data->nb_rx_queues)
 329                         j = 0;
 330                 reta = (reta << CHAR_BIT) | j;
 331                 if ((i & 3) == 3)
 332                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 333                                         rte_bswap32(reta));
 334         }
 335
 336         /*
 337          * Generate RSS hash based on packet types, TCP/UDP
 338          * port numbers and/or IPv4/v6 src and dst addresses
 339          */
 340         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 341         mrqc = 0;
 342         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 343         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 344         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 345         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 346         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 347         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 348         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 349         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 350         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 351
 352         if (mrqc == 0) {
 353                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 354                         "supported", hf);
 355                 return;
 356         }
 357
 358         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 359 }
 360
 361 static int
 362 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 363 {
 364         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 365         int i, ret;
 366         struct fm10k_tx_queue *txq;
 367         uint64_t base_addr;
 368         uint32_t size;
 369
 370         /* Disable TXINT to avoid possible interrupt */
 371         for (i = 0; i < hw->mac.max_queues; i++)
 372                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 373                                 3 << FM10K_TXINT_TIMER_SHIFT);
 374
 375         /* Setup TX queue */
 376         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 377                 txq = dev->data->tx_queues[i];
 378                 base_addr = txq->hw_ring_phys_addr;
 379                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 380
 381                 /* disable queue to avoid issues while updating state */
 382                 ret = tx_queue_disable(hw, i);
 383                 if (ret) {
 384                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 385                         return -1;
 386                 }
 387
 388                 /* set location and size for descriptor ring */
 389                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 390                                 base_addr & UINT64_LOWER_32BITS_MASK);
 391                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 392                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 393                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 394         }
 395         return 0;
 396 }
 397
 398 static int
 399 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 400 {
 401         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 402         int i, ret;
 403         struct fm10k_rx_queue *rxq;
 404         uint64_t base_addr;
 405         uint32_t size;
 406         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 407         uint16_t buf_size;
 408
 409         /* Disable RXINT to avoid possible interrupt */
 410         for (i = 0; i < hw->mac.max_queues; i++)
 411                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 412                                 3 << FM10K_RXINT_TIMER_SHIFT);
 413
 414         /* Setup RX queues */
 415         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 416                 rxq = dev->data->rx_queues[i];
 417                 base_addr = rxq->hw_ring_phys_addr;
 418                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 419
 420                 /* disable queue to avoid issues while updating state */
 421                 ret = rx_queue_disable(hw, i);
 422                 if (ret) {
 423                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 424                         return -1;
 425                 }
 426
 427                 /* Setup the Base and Length of the Rx Descriptor Ring */
 428                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 429                                 base_addr & UINT64_LOWER_32BITS_MASK);
 430                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 431                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 432                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 433
 434                 /* Configure the Rx buffer size for one buff without split */
 435                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 436                         RTE_PKTMBUF_HEADROOM);
 437                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 438                  * reserved for this purpose, and the worst case could be 511B.
 439                  * But SRR reg assumes all buffers have the same size. In order
 440                  * to fill the gap, we'll have to consider the worst case and
 441                  * assume 512B is reserved. If we don't do so, it's possible
 442                  * for HW to overwrite data to next mbuf.
 443                  */
 444                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 445
 446                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 447                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 448
 449                 /* It adds dual VLAN length for supporting dual VLAN */
 450                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 451                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 452                         dev->data->dev_conf.rxmode.enable_scatter) {
 453                         uint32_t reg;
 454                         dev->data->scattered_rx = 1;
 455                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 456                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 457                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 458                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 459                 }
 460
 461                 /* Enable drop on empty, it's RO for VF */
 462                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 463                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 464
 465                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 466                 FM10K_WRITE_FLUSH(hw);
 467         }
 468
 469         /* Configure RSS if applicable */
 470         fm10k_dev_mq_rx_configure(dev);
 471         return 0;
 472 }
 473
 474 static int
 475 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 476 {
 477         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 478         int err = -1;
 479         uint32_t reg;
 480         struct fm10k_rx_queue *rxq;
 481
 482         PMD_INIT_FUNC_TRACE();
 483
 484         if (rx_queue_id < dev->data->nb_rx_queues) {
 485                 rxq = dev->data->rx_queues[rx_queue_id];
 486                 err = rx_queue_reset(rxq);
 487                 if (err == -ENOMEM) {
 488                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 489                         return err;
 490                 } else if (err == -EINVAL) {
 491                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 492                                 " %d", err);
 493                         return err;
 494                 }
 495
 496                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 497                  * Note: this must be done AFTER the queue is enabled on real
 498                  * hardware, but BEFORE the queue is enabled when using the
 499                  * emulation platform. Do it in both places for now and remove
 500                  * this comment and the following two register writes when the
 501                  * emulation platform is no longer being used.
 502                  */
 503                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 504                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 505
 506                 /* Set PF ownership flag for PF devices */
 507                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 508                 if (hw->mac.type == fm10k_mac_pf)
 509                         reg |= FM10K_RXQCTL_PF;
 510                 reg |= FM10K_RXQCTL_ENABLE;
 511                 /* enable RX queue */
 512                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 513                 FM10K_WRITE_FLUSH(hw);
 514
 515                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 516                  * Note: this must be done AFTER the queue is enabled
 517                  */
 518                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 519                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 520         }
 521
 522         return err;
 523 }
 524
 525 static int
 526 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 527 {
 528         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 529
 530         PMD_INIT_FUNC_TRACE();
 531
 532         if (rx_queue_id < dev->data->nb_rx_queues) {
 533                 /* Disable RX queue */
 534                 rx_queue_disable(hw, rx_queue_id);
 535
 536                 /* Free mbuf and clean HW ring */
 537                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 538         }
 539
 540         return 0;
 541 }
 542
 543 static int
 544 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 545 {
 546         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 547         /** @todo - this should be defined in the shared code */
 548 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 549         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 550         int err = 0;
 551
 552         PMD_INIT_FUNC_TRACE();
 553
 554         if (tx_queue_id < dev->data->nb_tx_queues) {
 555                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 556
 557                 /* reset head and tail pointers */
 558                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 559                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 560
 561                 /* enable TX queue */
 562                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 563                                         FM10K_TXDCTL_ENABLE | txdctl);
 564                 FM10K_WRITE_FLUSH(hw);
 565         } else
 566                 err = -1;
 567
 568         return err;
 569 }
 570
 571 static int
 572 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 573 {
 574         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 575
 576         PMD_INIT_FUNC_TRACE();
 577
 578         if (tx_queue_id < dev->data->nb_tx_queues) {
 579                 tx_queue_disable(hw, tx_queue_id);
 580                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 581         }
 582
 583         return 0;
 584 }
 585
 586 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 587 {
 588         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 589                 != FM10K_DGLORTMAP_NONE);
 590 }
 591
 592 static void
 593 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 594 {
 595         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 596         int status;
 597
 598         PMD_INIT_FUNC_TRACE();
 599
 600         /* Return if it didn't acquire valid glort range */
 601         if (!fm10k_glort_valid(hw))
 602                 return;
 603
 604         fm10k_mbx_lock(hw);
 605         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 606                                 FM10K_XCAST_MODE_PROMISC);
 607         fm10k_mbx_unlock(hw);
 608
 609         if (status != FM10K_SUCCESS)
 610                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 611 }
 612
 613 static void
 614 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 615 {
 616         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 617         uint8_t mode;
 618         int status;
 619
 620         PMD_INIT_FUNC_TRACE();
 621
 622         /* Return if it didn't acquire valid glort range */
 623         if (!fm10k_glort_valid(hw))
 624                 return;
 625
 626         if (dev->data->all_multicast == 1)
 627                 mode = FM10K_XCAST_MODE_ALLMULTI;
 628         else
 629                 mode = FM10K_XCAST_MODE_NONE;
 630
 631         fm10k_mbx_lock(hw);
 632         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 633                                 mode);
 634         fm10k_mbx_unlock(hw);
 635
 636         if (status != FM10K_SUCCESS)
 637                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 638 }
 639
 640 static void
 641 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 642 {
 643         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 644         int status;
 645
 646         PMD_INIT_FUNC_TRACE();
 647
 648         /* Return if it didn't acquire valid glort range */
 649         if (!fm10k_glort_valid(hw))
 650                 return;
 651
 652         /* If promiscuous mode is enabled, it doesn't make sense to enable
 653          * allmulticast and disable promiscuous since fm10k only can select
 654          * one of the modes.
 655          */
 656         if (dev->data->promiscuous) {
 657                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 658                         "needn't enable allmulticast");
 659                 return;
 660         }
 661
 662         fm10k_mbx_lock(hw);
 663         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 664                                 FM10K_XCAST_MODE_ALLMULTI);
 665         fm10k_mbx_unlock(hw);
 666
 667         if (status != FM10K_SUCCESS)
 668                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 669 }
 670
 671 static void
 672 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 673 {
 674         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 675         int status;
 676
 677         PMD_INIT_FUNC_TRACE();
 678
 679         /* Return if it didn't acquire valid glort range */
 680         if (!fm10k_glort_valid(hw))
 681                 return;
 682
 683         if (dev->data->promiscuous) {
 684                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 685                         "since promisc mode is enabled");
 686                 return;
 687         }
 688
 689         fm10k_mbx_lock(hw);
 690         /* Change mode to unicast mode */
 691         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 692                                 FM10K_XCAST_MODE_NONE);
 693         fm10k_mbx_unlock(hw);
 694
 695         if (status != FM10K_SUCCESS)
 696                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 697 }
 698
 699 /* fls = find last set bit = 32 minus the number of leading zeros */
 700 #ifndef fls
 701 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 702 #endif
 703 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 704 static int
 705 fm10k_dev_start(struct rte_eth_dev *dev)
 706 {
 707         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 708         int i, diag;
 709
 710         PMD_INIT_FUNC_TRACE();
 711
 712         /* stop, init, then start the hw */
 713         diag = fm10k_stop_hw(hw);
 714         if (diag != FM10K_SUCCESS) {
 715                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 716                 return -EIO;
 717         }
 718
 719         diag = fm10k_init_hw(hw);
 720         if (diag != FM10K_SUCCESS) {
 721                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 722                 return -EIO;
 723         }
 724
 725         diag = fm10k_start_hw(hw);
 726         if (diag != FM10K_SUCCESS) {
 727                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 728                 return -EIO;
 729         }
 730
 731         diag = fm10k_dev_tx_init(dev);
 732         if (diag) {
 733                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 734                 return diag;
 735         }
 736
 737         diag = fm10k_dev_rx_init(dev);
 738         if (diag) {
 739                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 740                 return diag;
 741         }
 742
 743         if (hw->mac.type == fm10k_mac_pf) {
 744                 /* Establish only VSI 0 as valid */
 745                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 746
 747                 /* Configure RSS bits used in RETA table */
 748                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 749                                 fls(dev->data->nb_rx_queues - 1) <<
 750                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 751
 752                 /* Invalidate all other GLORT entries */
 753                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 754                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 755                                         FM10K_DGLORTMAP_NONE);
 756         }
 757
 758         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 759                 struct fm10k_rx_queue *rxq;
 760                 rxq = dev->data->rx_queues[i];
 761
 762                 if (rxq->rx_deferred_start)
 763                         continue;
 764                 diag = fm10k_dev_rx_queue_start(dev, i);
 765                 if (diag != 0) {
 766                         int j;
 767                         for (j = 0; j < i; ++j)
 768                                 rx_queue_clean(dev->data->rx_queues[j]);
 769                         return diag;
 770                 }
 771         }
 772
 773         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 774                 struct fm10k_tx_queue *txq;
 775                 txq = dev->data->tx_queues[i];
 776
 777                 if (txq->tx_deferred_start)
 778                         continue;
 779                 diag = fm10k_dev_tx_queue_start(dev, i);
 780                 if (diag != 0) {
 781                         int j;
 782                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 783                                 rx_queue_clean(dev->data->rx_queues[j]);
 784                         return diag;
 785                 }
 786         }
 787
 788         return 0;
 789 }
 790
 791 static void
 792 fm10k_dev_stop(struct rte_eth_dev *dev)
 793 {
 794         int i;
 795
 796         PMD_INIT_FUNC_TRACE();
 797
 798         for (i = 0; i < dev->data->nb_tx_queues; i++)
 799                 fm10k_dev_tx_queue_stop(dev, i);
 800
 801         for (i = 0; i < dev->data->nb_rx_queues; i++)
 802                 fm10k_dev_rx_queue_stop(dev, i);
 803 }
 804
 805 static void
 806 fm10k_dev_close(struct rte_eth_dev *dev)
 807 {
 808         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 809
 810         PMD_INIT_FUNC_TRACE();
 811
 812         /* Stop mailbox service first */
 813         fm10k_close_mbx_service(hw);
 814         fm10k_dev_stop(dev);
 815         fm10k_stop_hw(hw);
 816 }
 817
 818 static int
 819 fm10k_link_update(struct rte_eth_dev *dev,
 820         __rte_unused int wait_to_complete)
 821 {
 822         PMD_INIT_FUNC_TRACE();
 823
 824         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 825          * x8 PCIe interface. For now, we leave the speed undefined since there
 826          * is no 50Gbps Ethernet. */
 827         dev->data->dev_link.link_speed  = 0;
 828         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 829         dev->data->dev_link.link_status = 1;
 830
 831         return 0;
 832 }
 833
 834 static void
 835 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 836 {
 837         uint64_t ipackets, opackets, ibytes, obytes;
 838         struct fm10k_hw *hw =
 839                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 840         struct fm10k_hw_stats *hw_stats =
 841                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 842         int i;
 843
 844         PMD_INIT_FUNC_TRACE();
 845
 846         fm10k_update_hw_stats(hw, hw_stats);
 847
 848         ipackets = opackets = ibytes = obytes = 0;
 849         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 850                 (i < hw->mac.max_queues); ++i) {
 851                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 852                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 853                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 854                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 855                 ipackets += stats->q_ipackets[i];
 856                 opackets += stats->q_opackets[i];
 857                 ibytes   += stats->q_ibytes[i];
 858                 obytes   += stats->q_obytes[i];
 859         }
 860         stats->ipackets = ipackets;
 861         stats->opackets = opackets;
 862         stats->ibytes = ibytes;
 863         stats->obytes = obytes;
 864 }
 865
 866 static void
 867 fm10k_stats_reset(struct rte_eth_dev *dev)
 868 {
 869         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 870         struct fm10k_hw_stats *hw_stats =
 871                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 872
 873         PMD_INIT_FUNC_TRACE();
 874
 875         memset(hw_stats, 0, sizeof(*hw_stats));
 876         fm10k_rebind_hw_stats(hw, hw_stats);
 877 }
 878
 879 static void
 880 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 881         struct rte_eth_dev_info *dev_info)
 882 {
 883         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 884
 885         PMD_INIT_FUNC_TRACE();
 886
 887         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 888         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 889         dev_info->max_rx_queues      = hw->mac.max_queues;
 890         dev_info->max_tx_queues      = hw->mac.max_queues;
 891         dev_info->max_mac_addrs      = 1;
 892         dev_info->max_hash_mac_addrs = 0;
 893         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 894         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 895         dev_info->rx_offload_capa =
 896                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 897                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 898                 DEV_RX_OFFLOAD_TCP_CKSUM;
 899         dev_info->tx_offload_capa    = 0;
 900         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 901
 902         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 903                 .rx_thresh = {
 904                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 905                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 906                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 907                 },
 908                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 909                 .rx_drop_en = 0,
 910         };
 911
 912         dev_info->default_txconf = (struct rte_eth_txconf) {
 913                 .tx_thresh = {
 914                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 915                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 916                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 917                 },
 918                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 919                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 920                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 921                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 922         };
 923
 924 }
 925
 926 static int
 927 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 928 {
 929         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 930
 931         PMD_INIT_FUNC_TRACE();
 932
 933         /* @todo - add support for the VF */
 934         if (hw->mac.type != fm10k_mac_pf)
 935                 return -ENOTSUP;
 936
 937         return fm10k_update_vlan(hw, vlan_id, 0, on);
 938 }
 939
 940 static inline int
 941 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
 942 {
 943         if ((request < min) || (request > max) || ((request % mult) != 0))
 944                 return -1;
 945         else
 946                 return 0;
 947 }
 948
 949 /*
 950  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
 951  * the physical address is required. If the memzone is already created, then
 952  * this function returns a pointer to the existing memzone.
 953  */
 954 static inline const struct rte_memzone *
 955 allocate_hw_ring(const char *driver_name, const char *ring_name,
 956         uint8_t port_id, uint16_t queue_id, int socket_id,
 957         uint32_t size, uint32_t align)
 958 {
 959         char name[RTE_MEMZONE_NAMESIZE];
 960         const struct rte_memzone *mz;
 961
 962         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
 963                  driver_name, ring_name, port_id, queue_id, socket_id);
 964
 965         /* return the memzone if it already exists */
 966         mz = rte_memzone_lookup(name);
 967         if (mz)
 968                 return mz;
 969
 970 #ifdef RTE_LIBRTE_XEN_DOM0
 971         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
 972                                            RTE_PGSIZE_2M);
 973 #else
 974         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
 975 #endif
 976 }
 977
 978 static inline int
 979 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
 980 {
 981         if ((request < min) || (request > max) || ((div % request) != 0))
 982                 return -1;
 983         else
 984                 return 0;
 985 }
 986
 987 static inline int
 988 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
 989 {
 990         uint16_t rx_free_thresh;
 991
 992         if (conf->rx_free_thresh == 0)
 993                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
 994         else
 995                 rx_free_thresh = conf->rx_free_thresh;
 996
 997         /* make sure the requested threshold satisfies the constraints */
 998         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
 999                         FM10K_RX_FREE_THRESH_MAX(q),
1000                         FM10K_RX_FREE_THRESH_DIV(q),
1001                         rx_free_thresh)) {
1002                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1003                         "less than or equal to %u, "
1004                         "greater than or equal to %u, "
1005                         "and a divisor of %u",
1006                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1007                         FM10K_RX_FREE_THRESH_MIN(q),
1008                         FM10K_RX_FREE_THRESH_DIV(q));
1009                 return (-EINVAL);
1010         }
1011
1012         q->alloc_thresh = rx_free_thresh;
1013         q->drop_en = conf->rx_drop_en;
1014         q->rx_deferred_start = conf->rx_deferred_start;
1015
1016         return 0;
1017 }
1018
1019 /*
1020  * Hardware requires specific alignment for Rx packet buffers. At
1021  * least one of the following two conditions must be satisfied.
1022  *  1. Address is 512B aligned
1023  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1024  *
1025  * As such, the driver may need to adjust the DMA address within the
1026  * buffer by up to 512B.
1027  *
1028  * return 1 if the element size is valid, otherwise return 0.
1029  */
1030 static int
1031 mempool_element_size_valid(struct rte_mempool *mp)
1032 {
1033         uint32_t min_size;
1034
1035         /* elt_size includes mbuf header and headroom */
1036         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1037                         RTE_PKTMBUF_HEADROOM;
1038
1039         /* account for up to 512B of alignment */
1040         min_size -= FM10K_RX_DATABUF_ALIGN;
1041
1042         /* sanity check for overflow */
1043         if (min_size > mp->elt_size)
1044                 return 0;
1045
1046         /* size is valid */
1047         return 1;
1048 }
1049
1050 static int
1051 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1052         uint16_t nb_desc, unsigned int socket_id,
1053         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1054 {
1055         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1056         struct fm10k_rx_queue *q;
1057         const struct rte_memzone *mz;
1058
1059         PMD_INIT_FUNC_TRACE();
1060
1061         /* make sure the mempool element size can account for alignment. */
1062         if (!mempool_element_size_valid(mp)) {
1063                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1064                 return (-EINVAL);
1065         }
1066
1067         /* make sure a valid number of descriptors have been requested */
1068         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1069                                 FM10K_MULT_RX_DESC, nb_desc)) {
1070                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1071                         "less than or equal to %"PRIu32", "
1072                         "greater than or equal to %u, "
1073                         "and a multiple of %u",
1074                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1075                         FM10K_MULT_RX_DESC);
1076                 return (-EINVAL);
1077         }
1078
1079         /*
1080          * if this queue existed already, free the associated memory. The
1081          * queue cannot be reused in case we need to allocate memory on
1082          * different socket than was previously used.
1083          */
1084         if (dev->data->rx_queues[queue_id] != NULL) {
1085                 rx_queue_free(dev->data->rx_queues[queue_id]);
1086                 dev->data->rx_queues[queue_id] = NULL;
1087         }
1088
1089         /* allocate memory for the queue structure */
1090         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1091                                 socket_id);
1092         if (q == NULL) {
1093                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1094                 return (-ENOMEM);
1095         }
1096
1097         /* setup queue */
1098         q->mp = mp;
1099         q->nb_desc = nb_desc;
1100         q->port_id = dev->data->port_id;
1101         q->queue_id = queue_id;
1102         q->tail_ptr = (volatile uint32_t *)
1103                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1104         if (handle_rxconf(q, conf))
1105                 return (-EINVAL);
1106
1107         /* allocate memory for the software ring */
1108         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1109                                         nb_desc * sizeof(struct rte_mbuf *),
1110                                         RTE_CACHE_LINE_SIZE, socket_id);
1111         if (q->sw_ring == NULL) {
1112                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1113                 rte_free(q);
1114                 return (-ENOMEM);
1115         }
1116
1117         /*
1118          * allocate memory for the hardware descriptor ring. A memzone large
1119          * enough to hold the maximum ring size is requested to allow for
1120          * resizing in later calls to the queue setup function.
1121          */
1122         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1123                                 dev->data->port_id, queue_id, socket_id,
1124                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1125         if (mz == NULL) {
1126                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1127                 rte_free(q->sw_ring);
1128                 rte_free(q);
1129                 return (-ENOMEM);
1130         }
1131         q->hw_ring = mz->addr;
1132         q->hw_ring_phys_addr = mz->phys_addr;
1133
1134         dev->data->rx_queues[queue_id] = q;
1135         return 0;
1136 }
1137
1138 static void
1139 fm10k_rx_queue_release(void *queue)
1140 {
1141         PMD_INIT_FUNC_TRACE();
1142
1143         rx_queue_free(queue);
1144 }
1145
1146 static inline int
1147 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1148 {
1149         uint16_t tx_free_thresh;
1150         uint16_t tx_rs_thresh;
1151
1152         /* constraint MACROs require that tx_free_thresh is configured
1153          * before tx_rs_thresh */
1154         if (conf->tx_free_thresh == 0)
1155                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1156         else
1157                 tx_free_thresh = conf->tx_free_thresh;
1158
1159         /* make sure the requested threshold satisfies the constraints */
1160         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1161                         FM10K_TX_FREE_THRESH_MAX(q),
1162                         FM10K_TX_FREE_THRESH_DIV(q),
1163                         tx_free_thresh)) {
1164                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1165                         "less than or equal to %u, "
1166                         "greater than or equal to %u, "
1167                         "and a divisor of %u",
1168                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1169                         FM10K_TX_FREE_THRESH_MIN(q),
1170                         FM10K_TX_FREE_THRESH_DIV(q));
1171                 return (-EINVAL);
1172         }
1173
1174         q->free_thresh = tx_free_thresh;
1175
1176         if (conf->tx_rs_thresh == 0)
1177                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1178         else
1179                 tx_rs_thresh = conf->tx_rs_thresh;
1180
1181         q->tx_deferred_start = conf->tx_deferred_start;
1182
1183         /* make sure the requested threshold satisfies the constraints */
1184         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1185                         FM10K_TX_RS_THRESH_MAX(q),
1186                         FM10K_TX_RS_THRESH_DIV(q),
1187                         tx_rs_thresh)) {
1188                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1189                         "less than or equal to %u, "
1190                         "greater than or equal to %u, "
1191                         "and a divisor of %u",
1192                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1193                         FM10K_TX_RS_THRESH_MIN(q),
1194                         FM10K_TX_RS_THRESH_DIV(q));
1195                 return (-EINVAL);
1196         }
1197
1198         q->rs_thresh = tx_rs_thresh;
1199
1200         return 0;
1201 }
1202
1203 static int
1204 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1205         uint16_t nb_desc, unsigned int socket_id,
1206         const struct rte_eth_txconf *conf)
1207 {
1208         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1209         struct fm10k_tx_queue *q;
1210         const struct rte_memzone *mz;
1211
1212         PMD_INIT_FUNC_TRACE();
1213
1214         /* make sure a valid number of descriptors have been requested */
1215         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1216                                 FM10K_MULT_TX_DESC, nb_desc)) {
1217                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1218                         "less than or equal to %"PRIu32", "
1219                         "greater than or equal to %u, "
1220                         "and a multiple of %u",
1221                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1222                         FM10K_MULT_TX_DESC);
1223                 return (-EINVAL);
1224         }
1225
1226         /*
1227          * if this queue existed already, free the associated memory. The
1228          * queue cannot be reused in case we need to allocate memory on
1229          * different socket than was previously used.
1230          */
1231         if (dev->data->tx_queues[queue_id] != NULL) {
1232                 tx_queue_free(dev->data->tx_queues[queue_id]);
1233                 dev->data->tx_queues[queue_id] = NULL;
1234         }
1235
1236         /* allocate memory for the queue structure */
1237         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1238                                 socket_id);
1239         if (q == NULL) {
1240                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1241                 return (-ENOMEM);
1242         }
1243
1244         /* setup queue */
1245         q->nb_desc = nb_desc;
1246         q->port_id = dev->data->port_id;
1247         q->queue_id = queue_id;
1248         q->tail_ptr = (volatile uint32_t *)
1249                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1250         if (handle_txconf(q, conf))
1251                 return (-EINVAL);
1252
1253         /* allocate memory for the software ring */
1254         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1255                                         nb_desc * sizeof(struct rte_mbuf *),
1256                                         RTE_CACHE_LINE_SIZE, socket_id);
1257         if (q->sw_ring == NULL) {
1258                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1259                 rte_free(q);
1260                 return (-ENOMEM);
1261         }
1262
1263         /*
1264          * allocate memory for the hardware descriptor ring. A memzone large
1265          * enough to hold the maximum ring size is requested to allow for
1266          * resizing in later calls to the queue setup function.
1267          */
1268         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1269                                 dev->data->port_id, queue_id, socket_id,
1270                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1271         if (mz == NULL) {
1272                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1273                 rte_free(q->sw_ring);
1274                 rte_free(q);
1275                 return (-ENOMEM);
1276         }
1277         q->hw_ring = mz->addr;
1278         q->hw_ring_phys_addr = mz->phys_addr;
1279
1280         /*
1281          * allocate memory for the RS bit tracker. Enough slots to hold the
1282          * descriptor index for each RS bit needing to be set are required.
1283          */
1284         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1285                                 ((nb_desc + 1) / q->rs_thresh) *
1286                                 sizeof(uint16_t),
1287                                 RTE_CACHE_LINE_SIZE, socket_id);
1288         if (q->rs_tracker.list == NULL) {
1289                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1290                 rte_free(q->sw_ring);
1291                 rte_free(q);
1292                 return (-ENOMEM);
1293         }
1294
1295         dev->data->tx_queues[queue_id] = q;
1296         return 0;
1297 }
1298
1299 static void
1300 fm10k_tx_queue_release(void *queue)
1301 {
1302         PMD_INIT_FUNC_TRACE();
1303
1304         tx_queue_free(queue);
1305 }
1306
1307 static int
1308 fm10k_reta_update(struct rte_eth_dev *dev,
1309                         struct rte_eth_rss_reta_entry64 *reta_conf,
1310                         uint16_t reta_size)
1311 {
1312         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1313         uint16_t i, j, idx, shift;
1314         uint8_t mask;
1315         uint32_t reta;
1316
1317         PMD_INIT_FUNC_TRACE();
1318
1319         if (reta_size > FM10K_MAX_RSS_INDICES) {
1320                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1321                         "(%d) doesn't match the number hardware can supported "
1322                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1323                 return -EINVAL;
1324         }
1325
1326         /*
1327          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1328          * 128-entries in 32 registers
1329          */
1330         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1331                 idx = i / RTE_RETA_GROUP_SIZE;
1332                 shift = i % RTE_RETA_GROUP_SIZE;
1333                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1334                                 BIT_MASK_PER_UINT32);
1335                 if (mask == 0)
1336                         continue;
1337
1338                 reta = 0;
1339                 if (mask != BIT_MASK_PER_UINT32)
1340                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1341
1342                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1343                         if (mask & (0x1 << j)) {
1344                                 if (mask != 0xF)
1345                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1346                                 reta |= reta_conf[idx].reta[shift + j] <<
1347                                                 (CHAR_BIT * j);
1348                         }
1349                 }
1350                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1351         }
1352
1353         return 0;
1354 }
1355
1356 static int
1357 fm10k_reta_query(struct rte_eth_dev *dev,
1358                         struct rte_eth_rss_reta_entry64 *reta_conf,
1359                         uint16_t reta_size)
1360 {
1361         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1362         uint16_t i, j, idx, shift;
1363         uint8_t mask;
1364         uint32_t reta;
1365
1366         PMD_INIT_FUNC_TRACE();
1367
1368         if (reta_size < FM10K_MAX_RSS_INDICES) {
1369                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1370                         "(%d) doesn't match the number hardware can supported "
1371                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1372                 return -EINVAL;
1373         }
1374
1375         /*
1376          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1377          * 128-entries in 32 registers
1378          */
1379         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1380                 idx = i / RTE_RETA_GROUP_SIZE;
1381                 shift = i % RTE_RETA_GROUP_SIZE;
1382                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1383                                 BIT_MASK_PER_UINT32);
1384                 if (mask == 0)
1385                         continue;
1386
1387                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1388                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1389                         if (mask & (0x1 << j))
1390                                 reta_conf[idx].reta[shift + j] = ((reta >>
1391                                         CHAR_BIT * j) & UINT8_MAX);
1392                 }
1393         }
1394
1395         return 0;
1396 }
1397
1398 static int
1399 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1400         struct rte_eth_rss_conf *rss_conf)
1401 {
1402         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1403         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1404         uint32_t mrqc;
1405         uint64_t hf = rss_conf->rss_hf;
1406         int i;
1407
1408         PMD_INIT_FUNC_TRACE();
1409
1410         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1411                 FM10K_RSSRK_ENTRIES_PER_REG)
1412                 return -EINVAL;
1413
1414         if (hf == 0)
1415                 return -EINVAL;
1416
1417         mrqc = 0;
1418         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1419         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1420         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1421         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1422         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1423         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1424         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1425         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1426         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1427
1428         /* If the mapping doesn't fit any supported, return */
1429         if (mrqc == 0)
1430                 return -EINVAL;
1431
1432         if (key != NULL)
1433                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1434                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1435
1436         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1437
1438         return 0;
1439 }
1440
1441 static int
1442 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1443         struct rte_eth_rss_conf *rss_conf)
1444 {
1445         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1446         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1447         uint32_t mrqc;
1448         uint64_t hf;
1449         int i;
1450
1451         PMD_INIT_FUNC_TRACE();
1452
1453         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1454                                 FM10K_RSSRK_ENTRIES_PER_REG)
1455                 return -EINVAL;
1456
1457         if (key != NULL)
1458                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1459                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1460
1461         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1462         hf = 0;
1463         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1464         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1465         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1466         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1467         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1468         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1469         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1470         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1471         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1472
1473         rss_conf->rss_hf = hf;
1474
1475         return 0;
1476 }
1477
1478 static void
1479 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1480 {
1481         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1482         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1483
1484         /* Bind all local non-queue interrupt to vector 0 */
1485         int_map |= 0;
1486
1487         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1488         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1489         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1490         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1491         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1492         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1493
1494         /* Enable misc causes */
1495         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1496                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1497                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1498                                 FM10K_EIMR_ENABLE(MAILBOX) |
1499                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1500                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1501                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1502                                 FM10K_EIMR_ENABLE(VFLR));
1503
1504         /* Enable ITR 0 */
1505         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1506                                         FM10K_ITR_MASK_CLEAR);
1507         FM10K_WRITE_FLUSH(hw);
1508 }
1509
1510 static void
1511 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1512 {
1513         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1514         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1515
1516         /* Bind all local non-queue interrupt to vector 0 */
1517         int_map |= 0;
1518
1519         /* Only INT 0 available, other 15 are reserved. */
1520         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1521
1522         /* Enable ITR 0 */
1523         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1524                                         FM10K_ITR_MASK_CLEAR);
1525         FM10K_WRITE_FLUSH(hw);
1526 }
1527
1528 static int
1529 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1530 {
1531         struct fm10k_fault fault;
1532         int err;
1533         const char *estr = "Unknown error";
1534
1535         /* Process PCA fault */
1536         if (eicr & FM10K_EIMR_PCA_FAULT) {
1537                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1538                 if (err)
1539                         goto error;
1540                 switch (fault.type) {
1541                 case PCA_NO_FAULT:
1542                         estr = "PCA_NO_FAULT"; break;
1543                 case PCA_UNMAPPED_ADDR:
1544                         estr = "PCA_UNMAPPED_ADDR"; break;
1545                 case PCA_BAD_QACCESS_PF:
1546                         estr = "PCA_BAD_QACCESS_PF"; break;
1547                 case PCA_BAD_QACCESS_VF:
1548                         estr = "PCA_BAD_QACCESS_VF"; break;
1549                 case PCA_MALICIOUS_REQ:
1550                         estr = "PCA_MALICIOUS_REQ"; break;
1551                 case PCA_POISONED_TLP:
1552                         estr = "PCA_POISONED_TLP"; break;
1553                 case PCA_TLP_ABORT:
1554                         estr = "PCA_TLP_ABORT"; break;
1555                 default:
1556                         goto error;
1557                 }
1558                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1559                         estr, fault.func ? "VF" : "PF", fault.func,
1560                         fault.address, fault.specinfo);
1561         }
1562
1563         /* Process THI fault */
1564         if (eicr & FM10K_EIMR_THI_FAULT) {
1565                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1566                 if (err)
1567                         goto error;
1568                 switch (fault.type) {
1569                 case THI_NO_FAULT:
1570                         estr = "THI_NO_FAULT"; break;
1571                 case THI_MAL_DIS_Q_FAULT:
1572                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1573                 default:
1574                         goto error;
1575                 }
1576                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1577                         estr, fault.func ? "VF" : "PF", fault.func,
1578                         fault.address, fault.specinfo);
1579         }
1580
1581         /* Process FUM fault */
1582         if (eicr & FM10K_EIMR_FUM_FAULT) {
1583                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1584                 if (err)
1585                         goto error;
1586                 switch (fault.type) {
1587                 case FUM_NO_FAULT:
1588                         estr = "FUM_NO_FAULT"; break;
1589                 case FUM_UNMAPPED_ADDR:
1590                         estr = "FUM_UNMAPPED_ADDR"; break;
1591                 case FUM_POISONED_TLP:
1592                         estr = "FUM_POISONED_TLP"; break;
1593                 case FUM_BAD_VF_QACCESS:
1594                         estr = "FUM_BAD_VF_QACCESS"; break;
1595                 case FUM_ADD_DECODE_ERR:
1596                         estr = "FUM_ADD_DECODE_ERR"; break;
1597                 case FUM_RO_ERROR:
1598                         estr = "FUM_RO_ERROR"; break;
1599                 case FUM_QPRC_CRC_ERROR:
1600                         estr = "FUM_QPRC_CRC_ERROR"; break;
1601                 case FUM_CSR_TIMEOUT:
1602                         estr = "FUM_CSR_TIMEOUT"; break;
1603                 case FUM_INVALID_TYPE:
1604                         estr = "FUM_INVALID_TYPE"; break;
1605                 case FUM_INVALID_LENGTH:
1606                         estr = "FUM_INVALID_LENGTH"; break;
1607                 case FUM_INVALID_BE:
1608                         estr = "FUM_INVALID_BE"; break;
1609                 case FUM_INVALID_ALIGN:
1610                         estr = "FUM_INVALID_ALIGN"; break;
1611                 default:
1612                         goto error;
1613                 }
1614                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1615                         estr, fault.func ? "VF" : "PF", fault.func,
1616                         fault.address, fault.specinfo);
1617         }
1618
1619         if (estr)
1620                 return 0;
1621         return 0;
1622 error:
1623         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1624         return err;
1625 }
1626
1627 /**
1628  * PF interrupt handler triggered by NIC for handling specific interrupt.
1629  *
1630  * @param handle
1631  *  Pointer to interrupt handle.
1632  * @param param
1633  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1634  *
1635  * @return
1636  *  void
1637  */
1638 static void
1639 fm10k_dev_interrupt_handler_pf(
1640                         __rte_unused struct rte_intr_handle *handle,
1641                         void *param)
1642 {
1643         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1644         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1645         uint32_t cause, status;
1646
1647         if (hw->mac.type != fm10k_mac_pf)
1648                 return;
1649
1650         cause = FM10K_READ_REG(hw, FM10K_EICR);
1651
1652         /* Handle PCI fault cases */
1653         if (cause & FM10K_EICR_FAULT_MASK) {
1654                 PMD_INIT_LOG(ERR, "INT: find fault!");
1655                 fm10k_dev_handle_fault(hw, cause);
1656         }
1657
1658         /* Handle switch up/down */
1659         if (cause & FM10K_EICR_SWITCHNOTREADY)
1660                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1661
1662         if (cause & FM10K_EICR_SWITCHREADY)
1663                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1664
1665         /* Handle mailbox message */
1666         fm10k_mbx_lock(hw);
1667         hw->mbx.ops.process(hw, &hw->mbx);
1668         fm10k_mbx_unlock(hw);
1669
1670         /* Handle SRAM error */
1671         if (cause & FM10K_EICR_SRAMERROR) {
1672                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1673
1674                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1675                 /* Write to clear pending bits */
1676                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1677
1678                 /* Todo: print out error message after shared code  updates */
1679         }
1680
1681         /* Clear these 3 events if having any */
1682         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1683                  FM10K_EICR_SWITCHREADY;
1684         if (cause)
1685                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1686
1687         /* Re-enable interrupt from device side */
1688         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1689                                         FM10K_ITR_MASK_CLEAR);
1690         /* Re-enable interrupt from host side */
1691         rte_intr_enable(&(dev->pci_dev->intr_handle));
1692 }
1693
1694 /**
1695  * VF interrupt handler triggered by NIC for handling specific interrupt.
1696  *
1697  * @param handle
1698  *  Pointer to interrupt handle.
1699  * @param param
1700  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1701  *
1702  * @return
1703  *  void
1704  */
1705 static void
1706 fm10k_dev_interrupt_handler_vf(
1707                         __rte_unused struct rte_intr_handle *handle,
1708                         void *param)
1709 {
1710         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1711         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1712
1713         if (hw->mac.type != fm10k_mac_vf)
1714                 return;
1715
1716         /* Handle mailbox message if lock is acquired */
1717         fm10k_mbx_lock(hw);
1718         hw->mbx.ops.process(hw, &hw->mbx);
1719         fm10k_mbx_unlock(hw);
1720
1721         /* Re-enable interrupt from device side */
1722         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1723                                         FM10K_ITR_MASK_CLEAR);
1724         /* Re-enable interrupt from host side */
1725         rte_intr_enable(&(dev->pci_dev->intr_handle));
1726 }
1727
1728 /* Mailbox message handler in VF */
1729 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1730         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1731         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1732         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1733         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1734 };
1735
1736 /* Mailbox message handler in PF */
1737 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1738         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1739         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1740         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1741         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1742         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1743         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1744         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1745 };
1746
1747 static int
1748 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1749 {
1750         int err;
1751
1752         /* Initialize mailbox lock */
1753         fm10k_mbx_initlock(hw);
1754
1755         /* Replace default message handler with new ones */
1756         if (hw->mac.type == fm10k_mac_pf)
1757                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1758         else
1759                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1760
1761         if (err) {
1762                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1763                                 err);
1764                 return err;
1765         }
1766         /* Connect to SM for PF device or PF for VF device */
1767         return hw->mbx.ops.connect(hw, &hw->mbx);
1768 }
1769
1770 static void
1771 fm10k_close_mbx_service(struct fm10k_hw *hw)
1772 {
1773         /* Disconnect from SM for PF device or PF for VF device */
1774         hw->mbx.ops.disconnect(hw, &hw->mbx);
1775 }
1776
1777 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1778         .dev_configure          = fm10k_dev_configure,
1779         .dev_start              = fm10k_dev_start,
1780         .dev_stop               = fm10k_dev_stop,
1781         .dev_close              = fm10k_dev_close,
1782         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
1783         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
1784         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
1785         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
1786         .stats_get              = fm10k_stats_get,
1787         .stats_reset            = fm10k_stats_reset,
1788         .link_update            = fm10k_link_update,
1789         .dev_infos_get          = fm10k_dev_infos_get,
1790         .vlan_filter_set        = fm10k_vlan_filter_set,
1791         .rx_queue_start         = fm10k_dev_rx_queue_start,
1792         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1793         .tx_queue_start         = fm10k_dev_tx_queue_start,
1794         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1795         .rx_queue_setup         = fm10k_rx_queue_setup,
1796         .rx_queue_release       = fm10k_rx_queue_release,
1797         .tx_queue_setup         = fm10k_tx_queue_setup,
1798         .tx_queue_release       = fm10k_tx_queue_release,
1799         .reta_update            = fm10k_reta_update,
1800         .reta_query             = fm10k_reta_query,
1801         .rss_hash_update        = fm10k_rss_hash_update,
1802         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1803 };
1804
1805 static int
1806 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1807 {
1808         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1809         int diag;
1810
1811         PMD_INIT_FUNC_TRACE();
1812
1813         dev->dev_ops = &fm10k_eth_dev_ops;
1814         dev->rx_pkt_burst = &fm10k_recv_pkts;
1815         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1816
1817         if (dev->data->scattered_rx)
1818                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1819
1820         /* only initialize in the primary process */
1821         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1822                 return 0;
1823
1824         /* Vendor and Device ID need to be set before init of shared code */
1825         memset(hw, 0, sizeof(*hw));
1826         hw->device_id = dev->pci_dev->id.device_id;
1827         hw->vendor_id = dev->pci_dev->id.vendor_id;
1828         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1829         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1830         hw->revision_id = 0;
1831         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1832         if (hw->hw_addr == NULL) {
1833                 PMD_INIT_LOG(ERR, "Bad mem resource."
1834                         " Try to blacklist unused devices.");
1835                 return -EIO;
1836         }
1837
1838         /* Store fm10k_adapter pointer */
1839         hw->back = dev->data->dev_private;
1840
1841         /* Initialize the shared code */
1842         diag = fm10k_init_shared_code(hw);
1843         if (diag != FM10K_SUCCESS) {
1844                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1845                 return -EIO;
1846         }
1847
1848         /*
1849          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1850          * there is no way to get link status without reading BAR4.  Until this
1851          * works, assume we have maximum bandwidth.
1852          * @todo - fix bus info
1853          */
1854         hw->bus_caps.speed = fm10k_bus_speed_8000;
1855         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1856         hw->bus_caps.payload = fm10k_bus_payload_512;
1857         hw->bus.speed = fm10k_bus_speed_8000;
1858         hw->bus.width = fm10k_bus_width_pcie_x8;
1859         hw->bus.payload = fm10k_bus_payload_256;
1860
1861         /* Initialize the hw */
1862         diag = fm10k_init_hw(hw);
1863         if (diag != FM10K_SUCCESS) {
1864                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1865                 return -EIO;
1866         }
1867
1868         /* Initialize MAC address(es) */
1869         dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1870         if (dev->data->mac_addrs == NULL) {
1871                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1872                 return -ENOMEM;
1873         }
1874
1875         diag = fm10k_read_mac_addr(hw);
1876         if (diag != FM10K_SUCCESS) {
1877                 /*
1878                  * TODO: remove special handling on VF. Need shared code to
1879                  * fix first.
1880                  */
1881                 if (hw->mac.type == fm10k_mac_pf) {
1882                         PMD_INIT_LOG(ERR, "Read MAC addr failed: %d", diag);
1883                         return -EIO;
1884                 } else {
1885                         /* Generate a random addr */
1886                         eth_random_addr(hw->mac.addr);
1887                         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1888                 }
1889         }
1890
1891         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1892                         &dev->data->mac_addrs[0]);
1893
1894         /* Reset the hw statistics */
1895         fm10k_stats_reset(dev);
1896
1897         /* Reset the hw */
1898         diag = fm10k_reset_hw(hw);
1899         if (diag != FM10K_SUCCESS) {
1900                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1901                 return -EIO;
1902         }
1903
1904         /* Setup mailbox service */
1905         diag = fm10k_setup_mbx_service(hw);
1906         if (diag != FM10K_SUCCESS) {
1907                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1908                 return -EIO;
1909         }
1910
1911         /*PF/VF has different interrupt handling mechanism */
1912         if (hw->mac.type == fm10k_mac_pf) {
1913                 /* register callback func to eal lib */
1914                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1915                         fm10k_dev_interrupt_handler_pf, (void *)dev);
1916
1917                 /* enable MISC interrupt */
1918                 fm10k_dev_enable_intr_pf(dev);
1919         } else { /* VF */
1920                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1921                         fm10k_dev_interrupt_handler_vf, (void *)dev);
1922
1923                 fm10k_dev_enable_intr_vf(dev);
1924         }
1925
1926         /* Enable uio intr after callback registered */
1927         rte_intr_enable(&(dev->pci_dev->intr_handle));
1928
1929         hw->mac.ops.update_int_moderator(hw);
1930
1931         /* Make sure Switch Manager is ready before going forward. */
1932         if (hw->mac.type == fm10k_mac_pf) {
1933                 int switch_ready = 0;
1934                 int i;
1935
1936                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
1937                         fm10k_mbx_lock(hw);
1938                         hw->mac.ops.get_host_state(hw, &switch_ready);
1939                         fm10k_mbx_unlock(hw);
1940                         if (switch_ready)
1941                                 break;
1942                         /* Delay some time to acquire async LPORT_MAP info. */
1943                         rte_delay_us(WAIT_SWITCH_MSG_US);
1944                 }
1945
1946                 if (switch_ready == 0) {
1947                         PMD_INIT_LOG(ERR, "switch is not ready");
1948                         return -1;
1949                 }
1950         }
1951
1952         /*
1953          * Below function will trigger operations on mailbox, acquire lock to
1954          * avoid race condition from interrupt handler. Operations on mailbox
1955          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
1956          * will handle and generate an interrupt to our side. Then,  FIFO in
1957          * mailbox will be touched.
1958          */
1959         fm10k_mbx_lock(hw);
1960         /* Enable port first */
1961         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
1962
1963         /* Update default vlan */
1964         hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
1965
1966         /*
1967          * Add default mac/vlan filter. glort is assigned by SM for PF, while is
1968          * unused for VF. PF will assign correct glort for VF.
1969          */
1970         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
1971                               hw->mac.default_vid, 1, 0);
1972
1973         /* Set unicast mode by default. App can change to other mode in other
1974          * API func.
1975          */
1976         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1977                                         FM10K_XCAST_MODE_NONE);
1978
1979         fm10k_mbx_unlock(hw);
1980
1981
1982         return 0;
1983 }
1984
1985 /*
1986  * The set of PCI devices this driver supports. This driver will enable both PF
1987  * and SRIOV-VF devices.
1988  */
1989 static const struct rte_pci_id pci_id_fm10k_map[] = {
1990 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1991 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1992 #include "rte_pci_dev_ids.h"
1993         { .vendor_id = 0, /* sentinel */ },
1994 };
1995
1996 static struct eth_driver rte_pmd_fm10k = {
1997         .pci_drv = {
1998                 .name = "rte_pmd_fm10k",
1999                 .id_table = pci_id_fm10k_map,
2000                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
2001         },
2002         .eth_dev_init = eth_fm10k_dev_init,
2003         .dev_private_size = sizeof(struct fm10k_adapter),
2004 };
2005
2006 /*
2007  * Driver initialization routine.
2008  * Invoked once at EAL init time.
2009  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2010  */
2011 static int
2012 rte_pmd_fm10k_init(__rte_unused const char *name,
2013         __rte_unused const char *params)
2014 {
2015         PMD_INIT_FUNC_TRACE();
2016         rte_eth_driver_register(&rte_pmd_fm10k);
2017         return 0;
2018 }
2019
2020 static struct rte_driver rte_fm10k_driver = {
2021         .type = PMD_PDEV,
2022         .init = rte_pmd_fm10k_init,
2023 };
2024
2025 PMD_REGISTER_DRIVER(rte_fm10k_driver);