drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62
  63 static void
  64 fm10k_mbx_initlock(struct fm10k_hw *hw)
  65 {
  66         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  67 }
  68
  69 static void
  70 fm10k_mbx_lock(struct fm10k_hw *hw)
  71 {
  72         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  73                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  74 }
  75
  76 static void
  77 fm10k_mbx_unlock(struct fm10k_hw *hw)
  78 {
  79         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  80 }
  81
  82 /*
  83  * reset queue to initial state, allocate software buffers used when starting
  84  * device.
  85  * return 0 on success
  86  * return -ENOMEM if buffers cannot be allocated
  87  * return -EINVAL if buffers do not satisfy alignment condition
  88  */
  89 static inline int
  90 rx_queue_reset(struct fm10k_rx_queue *q)
  91 {
  92         uint64_t dma_addr;
  93         int i, diag;
  94         PMD_INIT_FUNC_TRACE();
  95
  96         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
  97         if (diag != 0)
  98                 return -ENOMEM;
  99
 100         for (i = 0; i < q->nb_desc; ++i) {
 101                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 102                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 103                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 104                                                 q->nb_desc);
 105                         return -EINVAL;
 106                 }
 107                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 108                 q->hw_ring[i].q.pkt_addr = dma_addr;
 109                 q->hw_ring[i].q.hdr_addr = dma_addr;
 110         }
 111
 112         q->next_dd = 0;
 113         q->next_alloc = 0;
 114         q->next_trigger = q->alloc_thresh - 1;
 115         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 116         return 0;
 117 }
 118
 119 /*
 120  * clean queue, descriptor rings, free software buffers used when stopping
 121  * device.
 122  */
 123 static inline void
 124 rx_queue_clean(struct fm10k_rx_queue *q)
 125 {
 126         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 127         uint32_t i;
 128         PMD_INIT_FUNC_TRACE();
 129
 130         /* zero descriptor rings */
 131         for (i = 0; i < q->nb_desc; ++i)
 132                 q->hw_ring[i] = zero;
 133
 134         /* free software buffers */
 135         for (i = 0; i < q->nb_desc; ++i) {
 136                 if (q->sw_ring[i]) {
 137                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 138                         q->sw_ring[i] = NULL;
 139                 }
 140         }
 141 }
 142
 143 /*
 144  * free all queue memory used when releasing the queue (i.e. configure)
 145  */
 146 static inline void
 147 rx_queue_free(struct fm10k_rx_queue *q)
 148 {
 149         PMD_INIT_FUNC_TRACE();
 150         if (q) {
 151                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 152                 rx_queue_clean(q);
 153                 if (q->sw_ring) {
 154                         rte_free(q->sw_ring);
 155                         q->sw_ring = NULL;
 156                 }
 157                 rte_free(q);
 158                 q = NULL;
 159         }
 160 }
 161
 162 /*
 163  * disable RX queue, wait unitl HW finished necessary flush operation
 164  */
 165 static inline int
 166 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 167 {
 168         uint32_t reg, i;
 169
 170         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 171         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 172                         reg & ~FM10K_RXQCTL_ENABLE);
 173
 174         /* Wait 100us at most */
 175         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 176                 rte_delay_us(1);
 177                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 178                 if (!(reg & FM10K_RXQCTL_ENABLE))
 179                         break;
 180         }
 181
 182         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 183                 return -1;
 184
 185         return 0;
 186 }
 187
 188 /*
 189  * reset queue to initial state, allocate software buffers used when starting
 190  * device
 191  */
 192 static inline void
 193 tx_queue_reset(struct fm10k_tx_queue *q)
 194 {
 195         PMD_INIT_FUNC_TRACE();
 196         q->last_free = 0;
 197         q->next_free = 0;
 198         q->nb_used = 0;
 199         q->nb_free = q->nb_desc - 1;
 200         q->free_trigger = q->nb_free - q->free_thresh;
 201         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 202         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 203 }
 204
 205 /*
 206  * clean queue, descriptor rings, free software buffers used when stopping
 207  * device
 208  */
 209 static inline void
 210 tx_queue_clean(struct fm10k_tx_queue *q)
 211 {
 212         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 213         uint32_t i;
 214         PMD_INIT_FUNC_TRACE();
 215
 216         /* zero descriptor rings */
 217         for (i = 0; i < q->nb_desc; ++i)
 218                 q->hw_ring[i] = zero;
 219
 220         /* free software buffers */
 221         for (i = 0; i < q->nb_desc; ++i) {
 222                 if (q->sw_ring[i]) {
 223                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 224                         q->sw_ring[i] = NULL;
 225                 }
 226         }
 227 }
 228
 229 /*
 230  * free all queue memory used when releasing the queue (i.e. configure)
 231  */
 232 static inline void
 233 tx_queue_free(struct fm10k_tx_queue *q)
 234 {
 235         PMD_INIT_FUNC_TRACE();
 236         if (q) {
 237                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 238                 tx_queue_clean(q);
 239                 if (q->rs_tracker.list) {
 240                         rte_free(q->rs_tracker.list);
 241                         q->rs_tracker.list = NULL;
 242                 }
 243                 if (q->sw_ring) {
 244                         rte_free(q->sw_ring);
 245                         q->sw_ring = NULL;
 246                 }
 247                 rte_free(q);
 248                 q = NULL;
 249         }
 250 }
 251
 252 /*
 253  * disable TX queue, wait unitl HW finished necessary flush operation
 254  */
 255 static inline int
 256 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 257 {
 258         uint32_t reg, i;
 259
 260         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 261         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 262                         reg & ~FM10K_TXDCTL_ENABLE);
 263
 264         /* Wait 100us at most */
 265         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 266                 rte_delay_us(1);
 267                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 268                 if (!(reg & FM10K_TXDCTL_ENABLE))
 269                         break;
 270         }
 271
 272         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 273                 return -1;
 274
 275         return 0;
 276 }
 277
 278 static int
 279 fm10k_dev_configure(struct rte_eth_dev *dev)
 280 {
 281         PMD_INIT_FUNC_TRACE();
 282
 283         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 284                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 285
 286         return 0;
 287 }
 288
 289 static void
 290 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 291 {
 292         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 293         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 294         uint32_t mrqc, *key, i, reta, j;
 295         uint64_t hf;
 296
 297 #define RSS_KEY_SIZE 40
 298         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 299                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 300                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 301                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 302                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 303                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 304         };
 305
 306         if (dev->data->nb_rx_queues == 1 ||
 307             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 308             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 309                 return;
 310
 311         /* random key is rss_intel_key (default) or user provided (rss_key) */
 312         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 313                 key = (uint32_t *)rss_intel_key;
 314         else
 315                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 316
 317         /* Now fill our hash function seeds, 4 bytes at a time */
 318         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 319                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 320
 321         /*
 322          * Fill in redirection table
 323          * The byte-swap is needed because NIC registers are in
 324          * little-endian order.
 325          */
 326         reta = 0;
 327         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 328                 if (j == dev->data->nb_rx_queues)
 329                         j = 0;
 330                 reta = (reta << CHAR_BIT) | j;
 331                 if ((i & 3) == 3)
 332                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 333                                         rte_bswap32(reta));
 334         }
 335
 336         /*
 337          * Generate RSS hash based on packet types, TCP/UDP
 338          * port numbers and/or IPv4/v6 src and dst addresses
 339          */
 340         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 341         mrqc = 0;
 342         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 343         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 344         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 345         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 346         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 347         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 348         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 349         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 350         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 351
 352         if (mrqc == 0) {
 353                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 354                         "supported", hf);
 355                 return;
 356         }
 357
 358         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 359 }
 360
 361 static int
 362 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 363 {
 364         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 365         int i, ret;
 366         struct fm10k_tx_queue *txq;
 367         uint64_t base_addr;
 368         uint32_t size;
 369
 370         /* Disable TXINT to avoid possible interrupt */
 371         for (i = 0; i < hw->mac.max_queues; i++)
 372                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 373                                 3 << FM10K_TXINT_TIMER_SHIFT);
 374
 375         /* Setup TX queue */
 376         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 377                 txq = dev->data->tx_queues[i];
 378                 base_addr = txq->hw_ring_phys_addr;
 379                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 380
 381                 /* disable queue to avoid issues while updating state */
 382                 ret = tx_queue_disable(hw, i);
 383                 if (ret) {
 384                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 385                         return -1;
 386                 }
 387
 388                 /* set location and size for descriptor ring */
 389                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 390                                 base_addr & UINT64_LOWER_32BITS_MASK);
 391                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 392                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 393                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 394         }
 395         return 0;
 396 }
 397
 398 static int
 399 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 400 {
 401         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 402         int i, ret;
 403         struct fm10k_rx_queue *rxq;
 404         uint64_t base_addr;
 405         uint32_t size;
 406         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 407         uint16_t buf_size;
 408
 409         /* Disable RXINT to avoid possible interrupt */
 410         for (i = 0; i < hw->mac.max_queues; i++)
 411                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 412                                 3 << FM10K_RXINT_TIMER_SHIFT);
 413
 414         /* Setup RX queues */
 415         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 416                 rxq = dev->data->rx_queues[i];
 417                 base_addr = rxq->hw_ring_phys_addr;
 418                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 419
 420                 /* disable queue to avoid issues while updating state */
 421                 ret = rx_queue_disable(hw, i);
 422                 if (ret) {
 423                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 424                         return -1;
 425                 }
 426
 427                 /* Setup the Base and Length of the Rx Descriptor Ring */
 428                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 429                                 base_addr & UINT64_LOWER_32BITS_MASK);
 430                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 431                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 432                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 433
 434                 /* Configure the Rx buffer size for one buff without split */
 435                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 436                         RTE_PKTMBUF_HEADROOM);
 437                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 438                  * reserved for this purpose, and the worst case could be 511B.
 439                  * But SRR reg assumes all buffers have the same size. In order
 440                  * to fill the gap, we'll have to consider the worst case and
 441                  * assume 512B is reserved. If we don't do so, it's possible
 442                  * for HW to overwrite data to next mbuf.
 443                  */
 444                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 445
 446                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 447                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 448
 449                 /* It adds dual VLAN length for supporting dual VLAN */
 450                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 451                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 452                         dev->data->dev_conf.rxmode.enable_scatter) {
 453                         uint32_t reg;
 454                         dev->data->scattered_rx = 1;
 455                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 456                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 457                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 458                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 459                 }
 460
 461                 /* Enable drop on empty, it's RO for VF */
 462                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 463                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 464
 465                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 466                 FM10K_WRITE_FLUSH(hw);
 467         }
 468
 469         /* Configure RSS if applicable */
 470         fm10k_dev_mq_rx_configure(dev);
 471         return 0;
 472 }
 473
 474 static int
 475 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 476 {
 477         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 478         int err = -1;
 479         uint32_t reg;
 480         struct fm10k_rx_queue *rxq;
 481
 482         PMD_INIT_FUNC_TRACE();
 483
 484         if (rx_queue_id < dev->data->nb_rx_queues) {
 485                 rxq = dev->data->rx_queues[rx_queue_id];
 486                 err = rx_queue_reset(rxq);
 487                 if (err == -ENOMEM) {
 488                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 489                         return err;
 490                 } else if (err == -EINVAL) {
 491                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 492                                 " %d", err);
 493                         return err;
 494                 }
 495
 496                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 497                  * Note: this must be done AFTER the queue is enabled on real
 498                  * hardware, but BEFORE the queue is enabled when using the
 499                  * emulation platform. Do it in both places for now and remove
 500                  * this comment and the following two register writes when the
 501                  * emulation platform is no longer being used.
 502                  */
 503                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 504                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 505
 506                 /* Set PF ownership flag for PF devices */
 507                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 508                 if (hw->mac.type == fm10k_mac_pf)
 509                         reg |= FM10K_RXQCTL_PF;
 510                 reg |= FM10K_RXQCTL_ENABLE;
 511                 /* enable RX queue */
 512                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 513                 FM10K_WRITE_FLUSH(hw);
 514
 515                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 516                  * Note: this must be done AFTER the queue is enabled
 517                  */
 518                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 519                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 520         }
 521
 522         return err;
 523 }
 524
 525 static int
 526 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 527 {
 528         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 529
 530         PMD_INIT_FUNC_TRACE();
 531
 532         if (rx_queue_id < dev->data->nb_rx_queues) {
 533                 /* Disable RX queue */
 534                 rx_queue_disable(hw, rx_queue_id);
 535
 536                 /* Free mbuf and clean HW ring */
 537                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 538         }
 539
 540         return 0;
 541 }
 542
 543 static int
 544 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 545 {
 546         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 547         /** @todo - this should be defined in the shared code */
 548 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 549         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 550         int err = 0;
 551
 552         PMD_INIT_FUNC_TRACE();
 553
 554         if (tx_queue_id < dev->data->nb_tx_queues) {
 555                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 556
 557                 /* reset head and tail pointers */
 558                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 559                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 560
 561                 /* enable TX queue */
 562                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 563                                         FM10K_TXDCTL_ENABLE | txdctl);
 564                 FM10K_WRITE_FLUSH(hw);
 565         } else
 566                 err = -1;
 567
 568         return err;
 569 }
 570
 571 static int
 572 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 573 {
 574         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 575
 576         PMD_INIT_FUNC_TRACE();
 577
 578         if (tx_queue_id < dev->data->nb_tx_queues) {
 579                 tx_queue_disable(hw, tx_queue_id);
 580                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 581         }
 582
 583         return 0;
 584 }
 585
 586 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 587 {
 588         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 589                 != FM10K_DGLORTMAP_NONE);
 590 }
 591
 592 static void
 593 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 594 {
 595         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 596         int status;
 597
 598         PMD_INIT_FUNC_TRACE();
 599
 600         /* Return if it didn't acquire valid glort range */
 601         if (!fm10k_glort_valid(hw))
 602                 return;
 603
 604         fm10k_mbx_lock(hw);
 605         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 606                                 FM10K_XCAST_MODE_PROMISC);
 607         fm10k_mbx_unlock(hw);
 608
 609         if (status != FM10K_SUCCESS)
 610                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 611 }
 612
 613 static void
 614 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 615 {
 616         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 617         uint8_t mode;
 618         int status;
 619
 620         PMD_INIT_FUNC_TRACE();
 621
 622         /* Return if it didn't acquire valid glort range */
 623         if (!fm10k_glort_valid(hw))
 624                 return;
 625
 626         if (dev->data->all_multicast == 1)
 627                 mode = FM10K_XCAST_MODE_ALLMULTI;
 628         else
 629                 mode = FM10K_XCAST_MODE_NONE;
 630
 631         fm10k_mbx_lock(hw);
 632         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 633                                 mode);
 634         fm10k_mbx_unlock(hw);
 635
 636         if (status != FM10K_SUCCESS)
 637                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 638 }
 639
 640 static void
 641 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 642 {
 643         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 644         int status;
 645
 646         PMD_INIT_FUNC_TRACE();
 647
 648         /* Return if it didn't acquire valid glort range */
 649         if (!fm10k_glort_valid(hw))
 650                 return;
 651
 652         /* If promiscuous mode is enabled, it doesn't make sense to enable
 653          * allmulticast and disable promiscuous since fm10k only can select
 654          * one of the modes.
 655          */
 656         if (dev->data->promiscuous) {
 657                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 658                         "needn't enable allmulticast");
 659                 return;
 660         }
 661
 662         fm10k_mbx_lock(hw);
 663         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 664                                 FM10K_XCAST_MODE_ALLMULTI);
 665         fm10k_mbx_unlock(hw);
 666
 667         if (status != FM10K_SUCCESS)
 668                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 669 }
 670
 671 static void
 672 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 673 {
 674         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 675         int status;
 676
 677         PMD_INIT_FUNC_TRACE();
 678
 679         /* Return if it didn't acquire valid glort range */
 680         if (!fm10k_glort_valid(hw))
 681                 return;
 682
 683         if (dev->data->promiscuous) {
 684                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 685                         "since promisc mode is enabled");
 686                 return;
 687         }
 688
 689         fm10k_mbx_lock(hw);
 690         /* Change mode to unicast mode */
 691         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 692                                 FM10K_XCAST_MODE_NONE);
 693         fm10k_mbx_unlock(hw);
 694
 695         if (status != FM10K_SUCCESS)
 696                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 697 }
 698
 699 /* fls = find last set bit = 32 minus the number of leading zeros */
 700 #ifndef fls
 701 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 702 #endif
 703 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 704 static int
 705 fm10k_dev_start(struct rte_eth_dev *dev)
 706 {
 707         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 708         int i, diag;
 709
 710         PMD_INIT_FUNC_TRACE();
 711
 712         /* stop, init, then start the hw */
 713         diag = fm10k_stop_hw(hw);
 714         if (diag != FM10K_SUCCESS) {
 715                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 716                 return -EIO;
 717         }
 718
 719         diag = fm10k_init_hw(hw);
 720         if (diag != FM10K_SUCCESS) {
 721                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 722                 return -EIO;
 723         }
 724
 725         diag = fm10k_start_hw(hw);
 726         if (diag != FM10K_SUCCESS) {
 727                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 728                 return -EIO;
 729         }
 730
 731         diag = fm10k_dev_tx_init(dev);
 732         if (diag) {
 733                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 734                 return diag;
 735         }
 736
 737         diag = fm10k_dev_rx_init(dev);
 738         if (diag) {
 739                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 740                 return diag;
 741         }
 742
 743         if (hw->mac.type == fm10k_mac_pf) {
 744                 /* Establish only VSI 0 as valid */
 745                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 746
 747                 /* Configure RSS bits used in RETA table */
 748                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 749                                 fls(dev->data->nb_rx_queues - 1) <<
 750                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 751
 752                 /* Invalidate all other GLORT entries */
 753                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 754                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 755                                         FM10K_DGLORTMAP_NONE);
 756         }
 757
 758         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 759                 struct fm10k_rx_queue *rxq;
 760                 rxq = dev->data->rx_queues[i];
 761
 762                 if (rxq->rx_deferred_start)
 763                         continue;
 764                 diag = fm10k_dev_rx_queue_start(dev, i);
 765                 if (diag != 0) {
 766                         int j;
 767                         for (j = 0; j < i; ++j)
 768                                 rx_queue_clean(dev->data->rx_queues[j]);
 769                         return diag;
 770                 }
 771         }
 772
 773         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 774                 struct fm10k_tx_queue *txq;
 775                 txq = dev->data->tx_queues[i];
 776
 777                 if (txq->tx_deferred_start)
 778                         continue;
 779                 diag = fm10k_dev_tx_queue_start(dev, i);
 780                 if (diag != 0) {
 781                         int j;
 782                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 783                                 rx_queue_clean(dev->data->rx_queues[j]);
 784                         return diag;
 785                 }
 786         }
 787
 788         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID) {
 789                 fm10k_mbx_lock(hw);
 790                 /* Update default vlan */
 791                 hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
 792
 793                 /* Add default mac/vlan filter to PF/Switch manager */
 794                 hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
 795                                 hw->mac.default_vid, true, 0);
 796                 fm10k_mbx_unlock(hw);
 797         }
 798
 799         return 0;
 800 }
 801
 802 static void
 803 fm10k_dev_stop(struct rte_eth_dev *dev)
 804 {
 805         int i;
 806
 807         PMD_INIT_FUNC_TRACE();
 808
 809         for (i = 0; i < dev->data->nb_tx_queues; i++)
 810                 fm10k_dev_tx_queue_stop(dev, i);
 811
 812         for (i = 0; i < dev->data->nb_rx_queues; i++)
 813                 fm10k_dev_rx_queue_stop(dev, i);
 814 }
 815
 816 static void
 817 fm10k_dev_close(struct rte_eth_dev *dev)
 818 {
 819         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 820
 821         PMD_INIT_FUNC_TRACE();
 822
 823         /* Stop mailbox service first */
 824         fm10k_close_mbx_service(hw);
 825         fm10k_dev_stop(dev);
 826         fm10k_stop_hw(hw);
 827 }
 828
 829 static int
 830 fm10k_link_update(struct rte_eth_dev *dev,
 831         __rte_unused int wait_to_complete)
 832 {
 833         PMD_INIT_FUNC_TRACE();
 834
 835         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 836          * x8 PCIe interface. For now, we leave the speed undefined since there
 837          * is no 50Gbps Ethernet. */
 838         dev->data->dev_link.link_speed  = 0;
 839         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 840         dev->data->dev_link.link_status = 1;
 841
 842         return 0;
 843 }
 844
 845 static void
 846 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 847 {
 848         uint64_t ipackets, opackets, ibytes, obytes;
 849         struct fm10k_hw *hw =
 850                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 851         struct fm10k_hw_stats *hw_stats =
 852                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 853         int i;
 854
 855         PMD_INIT_FUNC_TRACE();
 856
 857         fm10k_update_hw_stats(hw, hw_stats);
 858
 859         ipackets = opackets = ibytes = obytes = 0;
 860         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 861                 (i < hw->mac.max_queues); ++i) {
 862                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 863                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 864                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 865                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 866                 ipackets += stats->q_ipackets[i];
 867                 opackets += stats->q_opackets[i];
 868                 ibytes   += stats->q_ibytes[i];
 869                 obytes   += stats->q_obytes[i];
 870         }
 871         stats->ipackets = ipackets;
 872         stats->opackets = opackets;
 873         stats->ibytes = ibytes;
 874         stats->obytes = obytes;
 875 }
 876
 877 static void
 878 fm10k_stats_reset(struct rte_eth_dev *dev)
 879 {
 880         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 881         struct fm10k_hw_stats *hw_stats =
 882                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 883
 884         PMD_INIT_FUNC_TRACE();
 885
 886         memset(hw_stats, 0, sizeof(*hw_stats));
 887         fm10k_rebind_hw_stats(hw, hw_stats);
 888 }
 889
 890 static void
 891 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 892         struct rte_eth_dev_info *dev_info)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 899         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 900         dev_info->max_rx_queues      = hw->mac.max_queues;
 901         dev_info->max_tx_queues      = hw->mac.max_queues;
 902         dev_info->max_mac_addrs      = 1;
 903         dev_info->max_hash_mac_addrs = 0;
 904         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 905         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 906         dev_info->rx_offload_capa =
 907                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 908                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 909                 DEV_RX_OFFLOAD_TCP_CKSUM;
 910         dev_info->tx_offload_capa    = 0;
 911         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 912
 913         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 914                 .rx_thresh = {
 915                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 916                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 917                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 918                 },
 919                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 920                 .rx_drop_en = 0,
 921         };
 922
 923         dev_info->default_txconf = (struct rte_eth_txconf) {
 924                 .tx_thresh = {
 925                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 926                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 927                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 928                 },
 929                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 930                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 931                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 932                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 933         };
 934
 935 }
 936
 937 static int
 938 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 939 {
 940         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 941
 942         PMD_INIT_FUNC_TRACE();
 943
 944         /* @todo - add support for the VF */
 945         if (hw->mac.type != fm10k_mac_pf)
 946                 return -ENOTSUP;
 947
 948         return fm10k_update_vlan(hw, vlan_id, 0, on);
 949 }
 950
 951 static inline int
 952 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
 953 {
 954         if ((request < min) || (request > max) || ((request % mult) != 0))
 955                 return -1;
 956         else
 957                 return 0;
 958 }
 959
 960 /*
 961  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
 962  * the physical address is required. If the memzone is already created, then
 963  * this function returns a pointer to the existing memzone.
 964  */
 965 static inline const struct rte_memzone *
 966 allocate_hw_ring(const char *driver_name, const char *ring_name,
 967         uint8_t port_id, uint16_t queue_id, int socket_id,
 968         uint32_t size, uint32_t align)
 969 {
 970         char name[RTE_MEMZONE_NAMESIZE];
 971         const struct rte_memzone *mz;
 972
 973         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
 974                  driver_name, ring_name, port_id, queue_id, socket_id);
 975
 976         /* return the memzone if it already exists */
 977         mz = rte_memzone_lookup(name);
 978         if (mz)
 979                 return mz;
 980
 981 #ifdef RTE_LIBRTE_XEN_DOM0
 982         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
 983                                            RTE_PGSIZE_2M);
 984 #else
 985         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
 986 #endif
 987 }
 988
 989 static inline int
 990 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
 991 {
 992         if ((request < min) || (request > max) || ((div % request) != 0))
 993                 return -1;
 994         else
 995                 return 0;
 996 }
 997
 998 static inline int
 999 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1000 {
1001         uint16_t rx_free_thresh;
1002
1003         if (conf->rx_free_thresh == 0)
1004                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1005         else
1006                 rx_free_thresh = conf->rx_free_thresh;
1007
1008         /* make sure the requested threshold satisfies the constraints */
1009         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1010                         FM10K_RX_FREE_THRESH_MAX(q),
1011                         FM10K_RX_FREE_THRESH_DIV(q),
1012                         rx_free_thresh)) {
1013                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1014                         "less than or equal to %u, "
1015                         "greater than or equal to %u, "
1016                         "and a divisor of %u",
1017                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1018                         FM10K_RX_FREE_THRESH_MIN(q),
1019                         FM10K_RX_FREE_THRESH_DIV(q));
1020                 return (-EINVAL);
1021         }
1022
1023         q->alloc_thresh = rx_free_thresh;
1024         q->drop_en = conf->rx_drop_en;
1025         q->rx_deferred_start = conf->rx_deferred_start;
1026
1027         return 0;
1028 }
1029
1030 /*
1031  * Hardware requires specific alignment for Rx packet buffers. At
1032  * least one of the following two conditions must be satisfied.
1033  *  1. Address is 512B aligned
1034  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1035  *
1036  * As such, the driver may need to adjust the DMA address within the
1037  * buffer by up to 512B.
1038  *
1039  * return 1 if the element size is valid, otherwise return 0.
1040  */
1041 static int
1042 mempool_element_size_valid(struct rte_mempool *mp)
1043 {
1044         uint32_t min_size;
1045
1046         /* elt_size includes mbuf header and headroom */
1047         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1048                         RTE_PKTMBUF_HEADROOM;
1049
1050         /* account for up to 512B of alignment */
1051         min_size -= FM10K_RX_DATABUF_ALIGN;
1052
1053         /* sanity check for overflow */
1054         if (min_size > mp->elt_size)
1055                 return 0;
1056
1057         /* size is valid */
1058         return 1;
1059 }
1060
1061 static int
1062 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1063         uint16_t nb_desc, unsigned int socket_id,
1064         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1065 {
1066         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1067         struct fm10k_rx_queue *q;
1068         const struct rte_memzone *mz;
1069
1070         PMD_INIT_FUNC_TRACE();
1071
1072         /* make sure the mempool element size can account for alignment. */
1073         if (!mempool_element_size_valid(mp)) {
1074                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1075                 return (-EINVAL);
1076         }
1077
1078         /* make sure a valid number of descriptors have been requested */
1079         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1080                                 FM10K_MULT_RX_DESC, nb_desc)) {
1081                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1082                         "less than or equal to %"PRIu32", "
1083                         "greater than or equal to %u, "
1084                         "and a multiple of %u",
1085                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1086                         FM10K_MULT_RX_DESC);
1087                 return (-EINVAL);
1088         }
1089
1090         /*
1091          * if this queue existed already, free the associated memory. The
1092          * queue cannot be reused in case we need to allocate memory on
1093          * different socket than was previously used.
1094          */
1095         if (dev->data->rx_queues[queue_id] != NULL) {
1096                 rx_queue_free(dev->data->rx_queues[queue_id]);
1097                 dev->data->rx_queues[queue_id] = NULL;
1098         }
1099
1100         /* allocate memory for the queue structure */
1101         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1102                                 socket_id);
1103         if (q == NULL) {
1104                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1105                 return (-ENOMEM);
1106         }
1107
1108         /* setup queue */
1109         q->mp = mp;
1110         q->nb_desc = nb_desc;
1111         q->port_id = dev->data->port_id;
1112         q->queue_id = queue_id;
1113         q->tail_ptr = (volatile uint32_t *)
1114                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1115         if (handle_rxconf(q, conf))
1116                 return (-EINVAL);
1117
1118         /* allocate memory for the software ring */
1119         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1120                                         nb_desc * sizeof(struct rte_mbuf *),
1121                                         RTE_CACHE_LINE_SIZE, socket_id);
1122         if (q->sw_ring == NULL) {
1123                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1124                 rte_free(q);
1125                 return (-ENOMEM);
1126         }
1127
1128         /*
1129          * allocate memory for the hardware descriptor ring. A memzone large
1130          * enough to hold the maximum ring size is requested to allow for
1131          * resizing in later calls to the queue setup function.
1132          */
1133         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1134                                 dev->data->port_id, queue_id, socket_id,
1135                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1136         if (mz == NULL) {
1137                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1138                 rte_free(q->sw_ring);
1139                 rte_free(q);
1140                 return (-ENOMEM);
1141         }
1142         q->hw_ring = mz->addr;
1143         q->hw_ring_phys_addr = mz->phys_addr;
1144
1145         dev->data->rx_queues[queue_id] = q;
1146         return 0;
1147 }
1148
1149 static void
1150 fm10k_rx_queue_release(void *queue)
1151 {
1152         PMD_INIT_FUNC_TRACE();
1153
1154         rx_queue_free(queue);
1155 }
1156
1157 static inline int
1158 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1159 {
1160         uint16_t tx_free_thresh;
1161         uint16_t tx_rs_thresh;
1162
1163         /* constraint MACROs require that tx_free_thresh is configured
1164          * before tx_rs_thresh */
1165         if (conf->tx_free_thresh == 0)
1166                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1167         else
1168                 tx_free_thresh = conf->tx_free_thresh;
1169
1170         /* make sure the requested threshold satisfies the constraints */
1171         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1172                         FM10K_TX_FREE_THRESH_MAX(q),
1173                         FM10K_TX_FREE_THRESH_DIV(q),
1174                         tx_free_thresh)) {
1175                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1176                         "less than or equal to %u, "
1177                         "greater than or equal to %u, "
1178                         "and a divisor of %u",
1179                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1180                         FM10K_TX_FREE_THRESH_MIN(q),
1181                         FM10K_TX_FREE_THRESH_DIV(q));
1182                 return (-EINVAL);
1183         }
1184
1185         q->free_thresh = tx_free_thresh;
1186
1187         if (conf->tx_rs_thresh == 0)
1188                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1189         else
1190                 tx_rs_thresh = conf->tx_rs_thresh;
1191
1192         q->tx_deferred_start = conf->tx_deferred_start;
1193
1194         /* make sure the requested threshold satisfies the constraints */
1195         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1196                         FM10K_TX_RS_THRESH_MAX(q),
1197                         FM10K_TX_RS_THRESH_DIV(q),
1198                         tx_rs_thresh)) {
1199                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1200                         "less than or equal to %u, "
1201                         "greater than or equal to %u, "
1202                         "and a divisor of %u",
1203                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1204                         FM10K_TX_RS_THRESH_MIN(q),
1205                         FM10K_TX_RS_THRESH_DIV(q));
1206                 return (-EINVAL);
1207         }
1208
1209         q->rs_thresh = tx_rs_thresh;
1210
1211         return 0;
1212 }
1213
1214 static int
1215 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1216         uint16_t nb_desc, unsigned int socket_id,
1217         const struct rte_eth_txconf *conf)
1218 {
1219         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1220         struct fm10k_tx_queue *q;
1221         const struct rte_memzone *mz;
1222
1223         PMD_INIT_FUNC_TRACE();
1224
1225         /* make sure a valid number of descriptors have been requested */
1226         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1227                                 FM10K_MULT_TX_DESC, nb_desc)) {
1228                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1229                         "less than or equal to %"PRIu32", "
1230                         "greater than or equal to %u, "
1231                         "and a multiple of %u",
1232                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1233                         FM10K_MULT_TX_DESC);
1234                 return (-EINVAL);
1235         }
1236
1237         /*
1238          * if this queue existed already, free the associated memory. The
1239          * queue cannot be reused in case we need to allocate memory on
1240          * different socket than was previously used.
1241          */
1242         if (dev->data->tx_queues[queue_id] != NULL) {
1243                 tx_queue_free(dev->data->tx_queues[queue_id]);
1244                 dev->data->tx_queues[queue_id] = NULL;
1245         }
1246
1247         /* allocate memory for the queue structure */
1248         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1249                                 socket_id);
1250         if (q == NULL) {
1251                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1252                 return (-ENOMEM);
1253         }
1254
1255         /* setup queue */
1256         q->nb_desc = nb_desc;
1257         q->port_id = dev->data->port_id;
1258         q->queue_id = queue_id;
1259         q->tail_ptr = (volatile uint32_t *)
1260                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1261         if (handle_txconf(q, conf))
1262                 return (-EINVAL);
1263
1264         /* allocate memory for the software ring */
1265         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1266                                         nb_desc * sizeof(struct rte_mbuf *),
1267                                         RTE_CACHE_LINE_SIZE, socket_id);
1268         if (q->sw_ring == NULL) {
1269                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1270                 rte_free(q);
1271                 return (-ENOMEM);
1272         }
1273
1274         /*
1275          * allocate memory for the hardware descriptor ring. A memzone large
1276          * enough to hold the maximum ring size is requested to allow for
1277          * resizing in later calls to the queue setup function.
1278          */
1279         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1280                                 dev->data->port_id, queue_id, socket_id,
1281                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1282         if (mz == NULL) {
1283                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1284                 rte_free(q->sw_ring);
1285                 rte_free(q);
1286                 return (-ENOMEM);
1287         }
1288         q->hw_ring = mz->addr;
1289         q->hw_ring_phys_addr = mz->phys_addr;
1290
1291         /*
1292          * allocate memory for the RS bit tracker. Enough slots to hold the
1293          * descriptor index for each RS bit needing to be set are required.
1294          */
1295         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1296                                 ((nb_desc + 1) / q->rs_thresh) *
1297                                 sizeof(uint16_t),
1298                                 RTE_CACHE_LINE_SIZE, socket_id);
1299         if (q->rs_tracker.list == NULL) {
1300                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1301                 rte_free(q->sw_ring);
1302                 rte_free(q);
1303                 return (-ENOMEM);
1304         }
1305
1306         dev->data->tx_queues[queue_id] = q;
1307         return 0;
1308 }
1309
1310 static void
1311 fm10k_tx_queue_release(void *queue)
1312 {
1313         PMD_INIT_FUNC_TRACE();
1314
1315         tx_queue_free(queue);
1316 }
1317
1318 static int
1319 fm10k_reta_update(struct rte_eth_dev *dev,
1320                         struct rte_eth_rss_reta_entry64 *reta_conf,
1321                         uint16_t reta_size)
1322 {
1323         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1324         uint16_t i, j, idx, shift;
1325         uint8_t mask;
1326         uint32_t reta;
1327
1328         PMD_INIT_FUNC_TRACE();
1329
1330         if (reta_size > FM10K_MAX_RSS_INDICES) {
1331                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1332                         "(%d) doesn't match the number hardware can supported "
1333                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1334                 return -EINVAL;
1335         }
1336
1337         /*
1338          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1339          * 128-entries in 32 registers
1340          */
1341         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1342                 idx = i / RTE_RETA_GROUP_SIZE;
1343                 shift = i % RTE_RETA_GROUP_SIZE;
1344                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1345                                 BIT_MASK_PER_UINT32);
1346                 if (mask == 0)
1347                         continue;
1348
1349                 reta = 0;
1350                 if (mask != BIT_MASK_PER_UINT32)
1351                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1352
1353                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1354                         if (mask & (0x1 << j)) {
1355                                 if (mask != 0xF)
1356                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1357                                 reta |= reta_conf[idx].reta[shift + j] <<
1358                                                 (CHAR_BIT * j);
1359                         }
1360                 }
1361                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1362         }
1363
1364         return 0;
1365 }
1366
1367 static int
1368 fm10k_reta_query(struct rte_eth_dev *dev,
1369                         struct rte_eth_rss_reta_entry64 *reta_conf,
1370                         uint16_t reta_size)
1371 {
1372         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1373         uint16_t i, j, idx, shift;
1374         uint8_t mask;
1375         uint32_t reta;
1376
1377         PMD_INIT_FUNC_TRACE();
1378
1379         if (reta_size < FM10K_MAX_RSS_INDICES) {
1380                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1381                         "(%d) doesn't match the number hardware can supported "
1382                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1383                 return -EINVAL;
1384         }
1385
1386         /*
1387          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1388          * 128-entries in 32 registers
1389          */
1390         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1391                 idx = i / RTE_RETA_GROUP_SIZE;
1392                 shift = i % RTE_RETA_GROUP_SIZE;
1393                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1394                                 BIT_MASK_PER_UINT32);
1395                 if (mask == 0)
1396                         continue;
1397
1398                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1399                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1400                         if (mask & (0x1 << j))
1401                                 reta_conf[idx].reta[shift + j] = ((reta >>
1402                                         CHAR_BIT * j) & UINT8_MAX);
1403                 }
1404         }
1405
1406         return 0;
1407 }
1408
1409 static int
1410 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1411         struct rte_eth_rss_conf *rss_conf)
1412 {
1413         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1414         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1415         uint32_t mrqc;
1416         uint64_t hf = rss_conf->rss_hf;
1417         int i;
1418
1419         PMD_INIT_FUNC_TRACE();
1420
1421         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1422                 FM10K_RSSRK_ENTRIES_PER_REG)
1423                 return -EINVAL;
1424
1425         if (hf == 0)
1426                 return -EINVAL;
1427
1428         mrqc = 0;
1429         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1430         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1431         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1432         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1433         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1434         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1435         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1436         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1437         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1438
1439         /* If the mapping doesn't fit any supported, return */
1440         if (mrqc == 0)
1441                 return -EINVAL;
1442
1443         if (key != NULL)
1444                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1445                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1446
1447         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1448
1449         return 0;
1450 }
1451
1452 static int
1453 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1454         struct rte_eth_rss_conf *rss_conf)
1455 {
1456         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1457         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1458         uint32_t mrqc;
1459         uint64_t hf;
1460         int i;
1461
1462         PMD_INIT_FUNC_TRACE();
1463
1464         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1465                                 FM10K_RSSRK_ENTRIES_PER_REG)
1466                 return -EINVAL;
1467
1468         if (key != NULL)
1469                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1470                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1471
1472         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1473         hf = 0;
1474         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1475         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1476         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1477         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1478         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1479         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1480         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1481         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1482         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1483
1484         rss_conf->rss_hf = hf;
1485
1486         return 0;
1487 }
1488
1489 static void
1490 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1491 {
1492         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1493         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1494
1495         /* Bind all local non-queue interrupt to vector 0 */
1496         int_map |= 0;
1497
1498         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1499         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1500         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1501         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1502         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1503         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1504
1505         /* Enable misc causes */
1506         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1507                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1508                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1509                                 FM10K_EIMR_ENABLE(MAILBOX) |
1510                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1511                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1512                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1513                                 FM10K_EIMR_ENABLE(VFLR));
1514
1515         /* Enable ITR 0 */
1516         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1517                                         FM10K_ITR_MASK_CLEAR);
1518         FM10K_WRITE_FLUSH(hw);
1519 }
1520
1521 static void
1522 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1523 {
1524         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1525         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1526
1527         /* Bind all local non-queue interrupt to vector 0 */
1528         int_map |= 0;
1529
1530         /* Only INT 0 available, other 15 are reserved. */
1531         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1532
1533         /* Enable ITR 0 */
1534         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1535                                         FM10K_ITR_MASK_CLEAR);
1536         FM10K_WRITE_FLUSH(hw);
1537 }
1538
1539 static int
1540 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1541 {
1542         struct fm10k_fault fault;
1543         int err;
1544         const char *estr = "Unknown error";
1545
1546         /* Process PCA fault */
1547         if (eicr & FM10K_EIMR_PCA_FAULT) {
1548                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1549                 if (err)
1550                         goto error;
1551                 switch (fault.type) {
1552                 case PCA_NO_FAULT:
1553                         estr = "PCA_NO_FAULT"; break;
1554                 case PCA_UNMAPPED_ADDR:
1555                         estr = "PCA_UNMAPPED_ADDR"; break;
1556                 case PCA_BAD_QACCESS_PF:
1557                         estr = "PCA_BAD_QACCESS_PF"; break;
1558                 case PCA_BAD_QACCESS_VF:
1559                         estr = "PCA_BAD_QACCESS_VF"; break;
1560                 case PCA_MALICIOUS_REQ:
1561                         estr = "PCA_MALICIOUS_REQ"; break;
1562                 case PCA_POISONED_TLP:
1563                         estr = "PCA_POISONED_TLP"; break;
1564                 case PCA_TLP_ABORT:
1565                         estr = "PCA_TLP_ABORT"; break;
1566                 default:
1567                         goto error;
1568                 }
1569                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1570                         estr, fault.func ? "VF" : "PF", fault.func,
1571                         fault.address, fault.specinfo);
1572         }
1573
1574         /* Process THI fault */
1575         if (eicr & FM10K_EIMR_THI_FAULT) {
1576                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1577                 if (err)
1578                         goto error;
1579                 switch (fault.type) {
1580                 case THI_NO_FAULT:
1581                         estr = "THI_NO_FAULT"; break;
1582                 case THI_MAL_DIS_Q_FAULT:
1583                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1584                 default:
1585                         goto error;
1586                 }
1587                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1588                         estr, fault.func ? "VF" : "PF", fault.func,
1589                         fault.address, fault.specinfo);
1590         }
1591
1592         /* Process FUM fault */
1593         if (eicr & FM10K_EIMR_FUM_FAULT) {
1594                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1595                 if (err)
1596                         goto error;
1597                 switch (fault.type) {
1598                 case FUM_NO_FAULT:
1599                         estr = "FUM_NO_FAULT"; break;
1600                 case FUM_UNMAPPED_ADDR:
1601                         estr = "FUM_UNMAPPED_ADDR"; break;
1602                 case FUM_POISONED_TLP:
1603                         estr = "FUM_POISONED_TLP"; break;
1604                 case FUM_BAD_VF_QACCESS:
1605                         estr = "FUM_BAD_VF_QACCESS"; break;
1606                 case FUM_ADD_DECODE_ERR:
1607                         estr = "FUM_ADD_DECODE_ERR"; break;
1608                 case FUM_RO_ERROR:
1609                         estr = "FUM_RO_ERROR"; break;
1610                 case FUM_QPRC_CRC_ERROR:
1611                         estr = "FUM_QPRC_CRC_ERROR"; break;
1612                 case FUM_CSR_TIMEOUT:
1613                         estr = "FUM_CSR_TIMEOUT"; break;
1614                 case FUM_INVALID_TYPE:
1615                         estr = "FUM_INVALID_TYPE"; break;
1616                 case FUM_INVALID_LENGTH:
1617                         estr = "FUM_INVALID_LENGTH"; break;
1618                 case FUM_INVALID_BE:
1619                         estr = "FUM_INVALID_BE"; break;
1620                 case FUM_INVALID_ALIGN:
1621                         estr = "FUM_INVALID_ALIGN"; break;
1622                 default:
1623                         goto error;
1624                 }
1625                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1626                         estr, fault.func ? "VF" : "PF", fault.func,
1627                         fault.address, fault.specinfo);
1628         }
1629
1630         if (estr)
1631                 return 0;
1632         return 0;
1633 error:
1634         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1635         return err;
1636 }
1637
1638 /**
1639  * PF interrupt handler triggered by NIC for handling specific interrupt.
1640  *
1641  * @param handle
1642  *  Pointer to interrupt handle.
1643  * @param param
1644  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1645  *
1646  * @return
1647  *  void
1648  */
1649 static void
1650 fm10k_dev_interrupt_handler_pf(
1651                         __rte_unused struct rte_intr_handle *handle,
1652                         void *param)
1653 {
1654         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1655         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1656         uint32_t cause, status;
1657
1658         if (hw->mac.type != fm10k_mac_pf)
1659                 return;
1660
1661         cause = FM10K_READ_REG(hw, FM10K_EICR);
1662
1663         /* Handle PCI fault cases */
1664         if (cause & FM10K_EICR_FAULT_MASK) {
1665                 PMD_INIT_LOG(ERR, "INT: find fault!");
1666                 fm10k_dev_handle_fault(hw, cause);
1667         }
1668
1669         /* Handle switch up/down */
1670         if (cause & FM10K_EICR_SWITCHNOTREADY)
1671                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1672
1673         if (cause & FM10K_EICR_SWITCHREADY)
1674                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1675
1676         /* Handle mailbox message */
1677         fm10k_mbx_lock(hw);
1678         hw->mbx.ops.process(hw, &hw->mbx);
1679         fm10k_mbx_unlock(hw);
1680
1681         /* Handle SRAM error */
1682         if (cause & FM10K_EICR_SRAMERROR) {
1683                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1684
1685                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1686                 /* Write to clear pending bits */
1687                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1688
1689                 /* Todo: print out error message after shared code  updates */
1690         }
1691
1692         /* Clear these 3 events if having any */
1693         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1694                  FM10K_EICR_SWITCHREADY;
1695         if (cause)
1696                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1697
1698         /* Re-enable interrupt from device side */
1699         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1700                                         FM10K_ITR_MASK_CLEAR);
1701         /* Re-enable interrupt from host side */
1702         rte_intr_enable(&(dev->pci_dev->intr_handle));
1703 }
1704
1705 /**
1706  * VF interrupt handler triggered by NIC for handling specific interrupt.
1707  *
1708  * @param handle
1709  *  Pointer to interrupt handle.
1710  * @param param
1711  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1712  *
1713  * @return
1714  *  void
1715  */
1716 static void
1717 fm10k_dev_interrupt_handler_vf(
1718                         __rte_unused struct rte_intr_handle *handle,
1719                         void *param)
1720 {
1721         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1722         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1723
1724         if (hw->mac.type != fm10k_mac_vf)
1725                 return;
1726
1727         /* Handle mailbox message if lock is acquired */
1728         fm10k_mbx_lock(hw);
1729         hw->mbx.ops.process(hw, &hw->mbx);
1730         fm10k_mbx_unlock(hw);
1731
1732         /* Re-enable interrupt from device side */
1733         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1734                                         FM10K_ITR_MASK_CLEAR);
1735         /* Re-enable interrupt from host side */
1736         rte_intr_enable(&(dev->pci_dev->intr_handle));
1737 }
1738
1739 /* Mailbox message handler in VF */
1740 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1741         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1742         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1743         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1744         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1745 };
1746
1747 /* Mailbox message handler in PF */
1748 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1749         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1750         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1751         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1752         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1753         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1754         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1755         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1756 };
1757
1758 static int
1759 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1760 {
1761         int err;
1762
1763         /* Initialize mailbox lock */
1764         fm10k_mbx_initlock(hw);
1765
1766         /* Replace default message handler with new ones */
1767         if (hw->mac.type == fm10k_mac_pf)
1768                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1769         else
1770                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1771
1772         if (err) {
1773                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1774                                 err);
1775                 return err;
1776         }
1777         /* Connect to SM for PF device or PF for VF device */
1778         return hw->mbx.ops.connect(hw, &hw->mbx);
1779 }
1780
1781 static void
1782 fm10k_close_mbx_service(struct fm10k_hw *hw)
1783 {
1784         /* Disconnect from SM for PF device or PF for VF device */
1785         hw->mbx.ops.disconnect(hw, &hw->mbx);
1786 }
1787
1788 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1789         .dev_configure          = fm10k_dev_configure,
1790         .dev_start              = fm10k_dev_start,
1791         .dev_stop               = fm10k_dev_stop,
1792         .dev_close              = fm10k_dev_close,
1793         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
1794         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
1795         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
1796         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
1797         .stats_get              = fm10k_stats_get,
1798         .stats_reset            = fm10k_stats_reset,
1799         .link_update            = fm10k_link_update,
1800         .dev_infos_get          = fm10k_dev_infos_get,
1801         .vlan_filter_set        = fm10k_vlan_filter_set,
1802         .rx_queue_start         = fm10k_dev_rx_queue_start,
1803         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1804         .tx_queue_start         = fm10k_dev_tx_queue_start,
1805         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1806         .rx_queue_setup         = fm10k_rx_queue_setup,
1807         .rx_queue_release       = fm10k_rx_queue_release,
1808         .tx_queue_setup         = fm10k_tx_queue_setup,
1809         .tx_queue_release       = fm10k_tx_queue_release,
1810         .reta_update            = fm10k_reta_update,
1811         .reta_query             = fm10k_reta_query,
1812         .rss_hash_update        = fm10k_rss_hash_update,
1813         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1814 };
1815
1816 static int
1817 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1818 {
1819         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1820         int diag;
1821
1822         PMD_INIT_FUNC_TRACE();
1823
1824         dev->dev_ops = &fm10k_eth_dev_ops;
1825         dev->rx_pkt_burst = &fm10k_recv_pkts;
1826         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1827
1828         if (dev->data->scattered_rx)
1829                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1830
1831         /* only initialize in the primary process */
1832         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1833                 return 0;
1834
1835         /* Vendor and Device ID need to be set before init of shared code */
1836         memset(hw, 0, sizeof(*hw));
1837         hw->device_id = dev->pci_dev->id.device_id;
1838         hw->vendor_id = dev->pci_dev->id.vendor_id;
1839         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1840         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1841         hw->revision_id = 0;
1842         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1843         if (hw->hw_addr == NULL) {
1844                 PMD_INIT_LOG(ERR, "Bad mem resource."
1845                         " Try to blacklist unused devices.");
1846                 return -EIO;
1847         }
1848
1849         /* Store fm10k_adapter pointer */
1850         hw->back = dev->data->dev_private;
1851
1852         /* Initialize the shared code */
1853         diag = fm10k_init_shared_code(hw);
1854         if (diag != FM10K_SUCCESS) {
1855                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1856                 return -EIO;
1857         }
1858
1859         /*
1860          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1861          * there is no way to get link status without reading BAR4.  Until this
1862          * works, assume we have maximum bandwidth.
1863          * @todo - fix bus info
1864          */
1865         hw->bus_caps.speed = fm10k_bus_speed_8000;
1866         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1867         hw->bus_caps.payload = fm10k_bus_payload_512;
1868         hw->bus.speed = fm10k_bus_speed_8000;
1869         hw->bus.width = fm10k_bus_width_pcie_x8;
1870         hw->bus.payload = fm10k_bus_payload_256;
1871
1872         /* Initialize the hw */
1873         diag = fm10k_init_hw(hw);
1874         if (diag != FM10K_SUCCESS) {
1875                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1876                 return -EIO;
1877         }
1878
1879         /* Initialize MAC address(es) */
1880         dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1881         if (dev->data->mac_addrs == NULL) {
1882                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1883                 return -ENOMEM;
1884         }
1885
1886         diag = fm10k_read_mac_addr(hw);
1887
1888         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1889                         &dev->data->mac_addrs[0]);
1890
1891         if (diag != FM10K_SUCCESS ||
1892                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
1893
1894                 /* Generate a random addr */
1895                 eth_random_addr(hw->mac.addr);
1896                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1897                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1898                 &dev->data->mac_addrs[0]);
1899         }
1900
1901         /* Reset the hw statistics */
1902         fm10k_stats_reset(dev);
1903
1904         /* Reset the hw */
1905         diag = fm10k_reset_hw(hw);
1906         if (diag != FM10K_SUCCESS) {
1907                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1908                 return -EIO;
1909         }
1910
1911         /* Setup mailbox service */
1912         diag = fm10k_setup_mbx_service(hw);
1913         if (diag != FM10K_SUCCESS) {
1914                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1915                 return -EIO;
1916         }
1917
1918         /*PF/VF has different interrupt handling mechanism */
1919         if (hw->mac.type == fm10k_mac_pf) {
1920                 /* register callback func to eal lib */
1921                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1922                         fm10k_dev_interrupt_handler_pf, (void *)dev);
1923
1924                 /* enable MISC interrupt */
1925                 fm10k_dev_enable_intr_pf(dev);
1926         } else { /* VF */
1927                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1928                         fm10k_dev_interrupt_handler_vf, (void *)dev);
1929
1930                 fm10k_dev_enable_intr_vf(dev);
1931         }
1932
1933         /* Enable uio intr after callback registered */
1934         rte_intr_enable(&(dev->pci_dev->intr_handle));
1935
1936         hw->mac.ops.update_int_moderator(hw);
1937
1938         /* Make sure Switch Manager is ready before going forward. */
1939         if (hw->mac.type == fm10k_mac_pf) {
1940                 int switch_ready = 0;
1941                 int i;
1942
1943                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
1944                         fm10k_mbx_lock(hw);
1945                         hw->mac.ops.get_host_state(hw, &switch_ready);
1946                         fm10k_mbx_unlock(hw);
1947                         if (switch_ready)
1948                                 break;
1949                         /* Delay some time to acquire async LPORT_MAP info. */
1950                         rte_delay_us(WAIT_SWITCH_MSG_US);
1951                 }
1952
1953                 if (switch_ready == 0) {
1954                         PMD_INIT_LOG(ERR, "switch is not ready");
1955                         return -1;
1956                 }
1957         }
1958
1959         /*
1960          * Below function will trigger operations on mailbox, acquire lock to
1961          * avoid race condition from interrupt handler. Operations on mailbox
1962          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
1963          * will handle and generate an interrupt to our side. Then,  FIFO in
1964          * mailbox will be touched.
1965          */
1966         fm10k_mbx_lock(hw);
1967         /* Enable port first */
1968         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
1969
1970         /*
1971          * Add default mac. glort is assigned by SM for PF, while is
1972          * unused for VF. PF will assign correct glort for VF.
1973          */
1974         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
1975                                 0, 1, 0);
1976
1977         /* Set unicast mode by default. App can change to other mode in other
1978          * API func.
1979          */
1980         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1981                                         FM10K_XCAST_MODE_NONE);
1982
1983         fm10k_mbx_unlock(hw);
1984
1985
1986         return 0;
1987 }
1988
1989 /*
1990  * The set of PCI devices this driver supports. This driver will enable both PF
1991  * and SRIOV-VF devices.
1992  */
1993 static const struct rte_pci_id pci_id_fm10k_map[] = {
1994 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1995 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1996 #include "rte_pci_dev_ids.h"
1997         { .vendor_id = 0, /* sentinel */ },
1998 };
1999
2000 static struct eth_driver rte_pmd_fm10k = {
2001         .pci_drv = {
2002                 .name = "rte_pmd_fm10k",
2003                 .id_table = pci_id_fm10k_map,
2004                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
2005         },
2006         .eth_dev_init = eth_fm10k_dev_init,
2007         .dev_private_size = sizeof(struct fm10k_adapter),
2008 };
2009
2010 /*
2011  * Driver initialization routine.
2012  * Invoked once at EAL init time.
2013  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2014  */
2015 static int
2016 rte_pmd_fm10k_init(__rte_unused const char *name,
2017         __rte_unused const char *params)
2018 {
2019         PMD_INIT_FUNC_TRACE();
2020         rte_eth_driver_register(&rte_pmd_fm10k);
2021         return 0;
2022 }
2023
2024 static struct rte_driver rte_fm10k_driver = {
2025         .type = PMD_PDEV,
2026         .init = rte_pmd_fm10k_init,
2027 };
2028
2029 PMD_REGISTER_DRIVER(rte_fm10k_driver);