drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62
  63 static void
  64 fm10k_mbx_initlock(struct fm10k_hw *hw)
  65 {
  66         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  67 }
  68
  69 static void
  70 fm10k_mbx_lock(struct fm10k_hw *hw)
  71 {
  72         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  73                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  74 }
  75
  76 static void
  77 fm10k_mbx_unlock(struct fm10k_hw *hw)
  78 {
  79         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  80 }
  81
  82 /*
  83  * reset queue to initial state, allocate software buffers used when starting
  84  * device.
  85  * return 0 on success
  86  * return -ENOMEM if buffers cannot be allocated
  87  * return -EINVAL if buffers do not satisfy alignment condition
  88  */
  89 static inline int
  90 rx_queue_reset(struct fm10k_rx_queue *q)
  91 {
  92         uint64_t dma_addr;
  93         int i, diag;
  94         PMD_INIT_FUNC_TRACE();
  95
  96         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
  97         if (diag != 0)
  98                 return -ENOMEM;
  99
 100         for (i = 0; i < q->nb_desc; ++i) {
 101                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 102                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 103                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 104                                                 q->nb_desc);
 105                         return -EINVAL;
 106                 }
 107                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 108                 q->hw_ring[i].q.pkt_addr = dma_addr;
 109                 q->hw_ring[i].q.hdr_addr = dma_addr;
 110         }
 111
 112         q->next_dd = 0;
 113         q->next_alloc = 0;
 114         q->next_trigger = q->alloc_thresh - 1;
 115         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 116         return 0;
 117 }
 118
 119 /*
 120  * clean queue, descriptor rings, free software buffers used when stopping
 121  * device.
 122  */
 123 static inline void
 124 rx_queue_clean(struct fm10k_rx_queue *q)
 125 {
 126         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 127         uint32_t i;
 128         PMD_INIT_FUNC_TRACE();
 129
 130         /* zero descriptor rings */
 131         for (i = 0; i < q->nb_desc; ++i)
 132                 q->hw_ring[i] = zero;
 133
 134         /* free software buffers */
 135         for (i = 0; i < q->nb_desc; ++i) {
 136                 if (q->sw_ring[i]) {
 137                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 138                         q->sw_ring[i] = NULL;
 139                 }
 140         }
 141 }
 142
 143 /*
 144  * free all queue memory used when releasing the queue (i.e. configure)
 145  */
 146 static inline void
 147 rx_queue_free(struct fm10k_rx_queue *q)
 148 {
 149         PMD_INIT_FUNC_TRACE();
 150         if (q) {
 151                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 152                 rx_queue_clean(q);
 153                 if (q->sw_ring) {
 154                         rte_free(q->sw_ring);
 155                         q->sw_ring = NULL;
 156                 }
 157                 rte_free(q);
 158                 q = NULL;
 159         }
 160 }
 161
 162 /*
 163  * disable RX queue, wait unitl HW finished necessary flush operation
 164  */
 165 static inline int
 166 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 167 {
 168         uint32_t reg, i;
 169
 170         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 171         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 172                         reg & ~FM10K_RXQCTL_ENABLE);
 173
 174         /* Wait 100us at most */
 175         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 176                 rte_delay_us(1);
 177                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 178                 if (!(reg & FM10K_RXQCTL_ENABLE))
 179                         break;
 180         }
 181
 182         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 183                 return -1;
 184
 185         return 0;
 186 }
 187
 188 /*
 189  * reset queue to initial state, allocate software buffers used when starting
 190  * device
 191  */
 192 static inline void
 193 tx_queue_reset(struct fm10k_tx_queue *q)
 194 {
 195         PMD_INIT_FUNC_TRACE();
 196         q->last_free = 0;
 197         q->next_free = 0;
 198         q->nb_used = 0;
 199         q->nb_free = q->nb_desc - 1;
 200         q->free_trigger = q->nb_free - q->free_thresh;
 201         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 202         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 203 }
 204
 205 /*
 206  * clean queue, descriptor rings, free software buffers used when stopping
 207  * device
 208  */
 209 static inline void
 210 tx_queue_clean(struct fm10k_tx_queue *q)
 211 {
 212         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 213         uint32_t i;
 214         PMD_INIT_FUNC_TRACE();
 215
 216         /* zero descriptor rings */
 217         for (i = 0; i < q->nb_desc; ++i)
 218                 q->hw_ring[i] = zero;
 219
 220         /* free software buffers */
 221         for (i = 0; i < q->nb_desc; ++i) {
 222                 if (q->sw_ring[i]) {
 223                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 224                         q->sw_ring[i] = NULL;
 225                 }
 226         }
 227 }
 228
 229 /*
 230  * free all queue memory used when releasing the queue (i.e. configure)
 231  */
 232 static inline void
 233 tx_queue_free(struct fm10k_tx_queue *q)
 234 {
 235         PMD_INIT_FUNC_TRACE();
 236         if (q) {
 237                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 238                 tx_queue_clean(q);
 239                 if (q->rs_tracker.list) {
 240                         rte_free(q->rs_tracker.list);
 241                         q->rs_tracker.list = NULL;
 242                 }
 243                 if (q->sw_ring) {
 244                         rte_free(q->sw_ring);
 245                         q->sw_ring = NULL;
 246                 }
 247                 rte_free(q);
 248                 q = NULL;
 249         }
 250 }
 251
 252 /*
 253  * disable TX queue, wait unitl HW finished necessary flush operation
 254  */
 255 static inline int
 256 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 257 {
 258         uint32_t reg, i;
 259
 260         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 261         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 262                         reg & ~FM10K_TXDCTL_ENABLE);
 263
 264         /* Wait 100us at most */
 265         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 266                 rte_delay_us(1);
 267                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 268                 if (!(reg & FM10K_TXDCTL_ENABLE))
 269                         break;
 270         }
 271
 272         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 273                 return -1;
 274
 275         return 0;
 276 }
 277
 278 static int
 279 fm10k_dev_configure(struct rte_eth_dev *dev)
 280 {
 281         PMD_INIT_FUNC_TRACE();
 282
 283         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 284                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 285
 286         return 0;
 287 }
 288
 289 static void
 290 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 291 {
 292         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 293         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 294         uint32_t mrqc, *key, i, reta, j;
 295         uint64_t hf;
 296
 297 #define RSS_KEY_SIZE 40
 298         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 299                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 300                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 301                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 302                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 303                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 304         };
 305
 306         if (dev->data->nb_rx_queues == 1 ||
 307             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 308             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 309                 return;
 310
 311         /* random key is rss_intel_key (default) or user provided (rss_key) */
 312         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 313                 key = (uint32_t *)rss_intel_key;
 314         else
 315                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 316
 317         /* Now fill our hash function seeds, 4 bytes at a time */
 318         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 319                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 320
 321         /*
 322          * Fill in redirection table
 323          * The byte-swap is needed because NIC registers are in
 324          * little-endian order.
 325          */
 326         reta = 0;
 327         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 328                 if (j == dev->data->nb_rx_queues)
 329                         j = 0;
 330                 reta = (reta << CHAR_BIT) | j;
 331                 if ((i & 3) == 3)
 332                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 333                                         rte_bswap32(reta));
 334         }
 335
 336         /*
 337          * Generate RSS hash based on packet types, TCP/UDP
 338          * port numbers and/or IPv4/v6 src and dst addresses
 339          */
 340         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 341         mrqc = 0;
 342         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 343         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 344         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 345         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 346         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 347         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 348         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 349         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 350         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 351
 352         if (mrqc == 0) {
 353                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 354                         "supported", hf);
 355                 return;
 356         }
 357
 358         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 359 }
 360
 361 static int
 362 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 363 {
 364         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 365         int i, ret;
 366         struct fm10k_tx_queue *txq;
 367         uint64_t base_addr;
 368         uint32_t size;
 369
 370         /* Disable TXINT to avoid possible interrupt */
 371         for (i = 0; i < hw->mac.max_queues; i++)
 372                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 373                                 3 << FM10K_TXINT_TIMER_SHIFT);
 374
 375         /* Setup TX queue */
 376         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 377                 txq = dev->data->tx_queues[i];
 378                 base_addr = txq->hw_ring_phys_addr;
 379                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 380
 381                 /* disable queue to avoid issues while updating state */
 382                 ret = tx_queue_disable(hw, i);
 383                 if (ret) {
 384                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 385                         return -1;
 386                 }
 387
 388                 /* set location and size for descriptor ring */
 389                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 390                                 base_addr & UINT64_LOWER_32BITS_MASK);
 391                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 392                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 393                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 394         }
 395         return 0;
 396 }
 397
 398 static int
 399 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 400 {
 401         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 402         int i, ret;
 403         struct fm10k_rx_queue *rxq;
 404         uint64_t base_addr;
 405         uint32_t size;
 406         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 407         uint16_t buf_size;
 408
 409         /* Disable RXINT to avoid possible interrupt */
 410         for (i = 0; i < hw->mac.max_queues; i++)
 411                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 412                                 3 << FM10K_RXINT_TIMER_SHIFT);
 413
 414         /* Setup RX queues */
 415         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 416                 rxq = dev->data->rx_queues[i];
 417                 base_addr = rxq->hw_ring_phys_addr;
 418                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 419
 420                 /* disable queue to avoid issues while updating state */
 421                 ret = rx_queue_disable(hw, i);
 422                 if (ret) {
 423                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 424                         return -1;
 425                 }
 426
 427                 /* Setup the Base and Length of the Rx Descriptor Ring */
 428                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 429                                 base_addr & UINT64_LOWER_32BITS_MASK);
 430                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 431                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 432                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 433
 434                 /* Configure the Rx buffer size for one buff without split */
 435                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 436                         RTE_PKTMBUF_HEADROOM);
 437                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 438                  * reserved for this purpose, and the worst case could be 511B.
 439                  * But SRR reg assumes all buffers have the same size. In order
 440                  * to fill the gap, we'll have to consider the worst case and
 441                  * assume 512B is reserved. If we don't do so, it's possible
 442                  * for HW to overwrite data to next mbuf.
 443                  */
 444                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 445
 446                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 447                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 448
 449                 /* It adds dual VLAN length for supporting dual VLAN */
 450                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 451                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 452                         dev->data->dev_conf.rxmode.enable_scatter) {
 453                         uint32_t reg;
 454                         dev->data->scattered_rx = 1;
 455                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 456                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 457                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 458                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 459                 }
 460
 461                 /* Enable drop on empty, it's RO for VF */
 462                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 463                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 464
 465                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 466                 FM10K_WRITE_FLUSH(hw);
 467         }
 468
 469         /* Configure RSS if applicable */
 470         fm10k_dev_mq_rx_configure(dev);
 471         return 0;
 472 }
 473
 474 static int
 475 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 476 {
 477         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 478         int err = -1;
 479         uint32_t reg;
 480         struct fm10k_rx_queue *rxq;
 481
 482         PMD_INIT_FUNC_TRACE();
 483
 484         if (rx_queue_id < dev->data->nb_rx_queues) {
 485                 rxq = dev->data->rx_queues[rx_queue_id];
 486                 err = rx_queue_reset(rxq);
 487                 if (err == -ENOMEM) {
 488                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 489                         return err;
 490                 } else if (err == -EINVAL) {
 491                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 492                                 " %d", err);
 493                         return err;
 494                 }
 495
 496                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 497                  * Note: this must be done AFTER the queue is enabled on real
 498                  * hardware, but BEFORE the queue is enabled when using the
 499                  * emulation platform. Do it in both places for now and remove
 500                  * this comment and the following two register writes when the
 501                  * emulation platform is no longer being used.
 502                  */
 503                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 504                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 505
 506                 /* Set PF ownership flag for PF devices */
 507                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 508                 if (hw->mac.type == fm10k_mac_pf)
 509                         reg |= FM10K_RXQCTL_PF;
 510                 reg |= FM10K_RXQCTL_ENABLE;
 511                 /* enable RX queue */
 512                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 513                 FM10K_WRITE_FLUSH(hw);
 514
 515                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 516                  * Note: this must be done AFTER the queue is enabled
 517                  */
 518                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 519                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 520         }
 521
 522         return err;
 523 }
 524
 525 static int
 526 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 527 {
 528         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 529
 530         PMD_INIT_FUNC_TRACE();
 531
 532         if (rx_queue_id < dev->data->nb_rx_queues) {
 533                 /* Disable RX queue */
 534                 rx_queue_disable(hw, rx_queue_id);
 535
 536                 /* Free mbuf and clean HW ring */
 537                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 538         }
 539
 540         return 0;
 541 }
 542
 543 static int
 544 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 545 {
 546         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 547         /** @todo - this should be defined in the shared code */
 548 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 549         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 550         int err = 0;
 551
 552         PMD_INIT_FUNC_TRACE();
 553
 554         if (tx_queue_id < dev->data->nb_tx_queues) {
 555                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 556
 557                 /* reset head and tail pointers */
 558                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 559                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 560
 561                 /* enable TX queue */
 562                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 563                                         FM10K_TXDCTL_ENABLE | txdctl);
 564                 FM10K_WRITE_FLUSH(hw);
 565         } else
 566                 err = -1;
 567
 568         return err;
 569 }
 570
 571 static int
 572 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 573 {
 574         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 575
 576         PMD_INIT_FUNC_TRACE();
 577
 578         if (tx_queue_id < dev->data->nb_tx_queues) {
 579                 tx_queue_disable(hw, tx_queue_id);
 580                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 581         }
 582
 583         return 0;
 584 }
 585
 586 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 587 {
 588         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 589                 != FM10K_DGLORTMAP_NONE);
 590 }
 591
 592 static void
 593 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 594 {
 595         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 596         int status;
 597
 598         PMD_INIT_FUNC_TRACE();
 599
 600         /* Return if it didn't acquire valid glort range */
 601         if (!fm10k_glort_valid(hw))
 602                 return;
 603
 604         fm10k_mbx_lock(hw);
 605         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 606                                 FM10K_XCAST_MODE_PROMISC);
 607         fm10k_mbx_unlock(hw);
 608
 609         if (status != FM10K_SUCCESS)
 610                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 611 }
 612
 613 static void
 614 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 615 {
 616         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 617         uint8_t mode;
 618         int status;
 619
 620         PMD_INIT_FUNC_TRACE();
 621
 622         /* Return if it didn't acquire valid glort range */
 623         if (!fm10k_glort_valid(hw))
 624                 return;
 625
 626         if (dev->data->all_multicast == 1)
 627                 mode = FM10K_XCAST_MODE_ALLMULTI;
 628         else
 629                 mode = FM10K_XCAST_MODE_NONE;
 630
 631         fm10k_mbx_lock(hw);
 632         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 633                                 mode);
 634         fm10k_mbx_unlock(hw);
 635
 636         if (status != FM10K_SUCCESS)
 637                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 638 }
 639
 640 static void
 641 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 642 {
 643         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 644         int status;
 645
 646         PMD_INIT_FUNC_TRACE();
 647
 648         /* Return if it didn't acquire valid glort range */
 649         if (!fm10k_glort_valid(hw))
 650                 return;
 651
 652         /* If promiscuous mode is enabled, it doesn't make sense to enable
 653          * allmulticast and disable promiscuous since fm10k only can select
 654          * one of the modes.
 655          */
 656         if (dev->data->promiscuous) {
 657                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 658                         "needn't enable allmulticast");
 659                 return;
 660         }
 661
 662         fm10k_mbx_lock(hw);
 663         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 664                                 FM10K_XCAST_MODE_ALLMULTI);
 665         fm10k_mbx_unlock(hw);
 666
 667         if (status != FM10K_SUCCESS)
 668                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 669 }
 670
 671 static void
 672 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 673 {
 674         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 675         int status;
 676
 677         PMD_INIT_FUNC_TRACE();
 678
 679         /* Return if it didn't acquire valid glort range */
 680         if (!fm10k_glort_valid(hw))
 681                 return;
 682
 683         if (dev->data->promiscuous) {
 684                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 685                         "since promisc mode is enabled");
 686                 return;
 687         }
 688
 689         fm10k_mbx_lock(hw);
 690         /* Change mode to unicast mode */
 691         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 692                                 FM10K_XCAST_MODE_NONE);
 693         fm10k_mbx_unlock(hw);
 694
 695         if (status != FM10K_SUCCESS)
 696                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 697 }
 698
 699 /* fls = find last set bit = 32 minus the number of leading zeros */
 700 #ifndef fls
 701 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 702 #endif
 703 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 704 static int
 705 fm10k_dev_start(struct rte_eth_dev *dev)
 706 {
 707         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 708         int i, diag;
 709
 710         PMD_INIT_FUNC_TRACE();
 711
 712         /* stop, init, then start the hw */
 713         diag = fm10k_stop_hw(hw);
 714         if (diag != FM10K_SUCCESS) {
 715                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 716                 return -EIO;
 717         }
 718
 719         diag = fm10k_init_hw(hw);
 720         if (diag != FM10K_SUCCESS) {
 721                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 722                 return -EIO;
 723         }
 724
 725         diag = fm10k_start_hw(hw);
 726         if (diag != FM10K_SUCCESS) {
 727                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 728                 return -EIO;
 729         }
 730
 731         diag = fm10k_dev_tx_init(dev);
 732         if (diag) {
 733                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 734                 return diag;
 735         }
 736
 737         diag = fm10k_dev_rx_init(dev);
 738         if (diag) {
 739                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 740                 return diag;
 741         }
 742
 743         if (hw->mac.type == fm10k_mac_pf) {
 744                 /* Establish only VSI 0 as valid */
 745                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 746
 747                 /* Configure RSS bits used in RETA table */
 748                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 749                                 fls(dev->data->nb_rx_queues - 1) <<
 750                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 751
 752                 /* Invalidate all other GLORT entries */
 753                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 754                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 755                                         FM10K_DGLORTMAP_NONE);
 756         }
 757
 758         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 759                 struct fm10k_rx_queue *rxq;
 760                 rxq = dev->data->rx_queues[i];
 761
 762                 if (rxq->rx_deferred_start)
 763                         continue;
 764                 diag = fm10k_dev_rx_queue_start(dev, i);
 765                 if (diag != 0) {
 766                         int j;
 767                         for (j = 0; j < i; ++j)
 768                                 rx_queue_clean(dev->data->rx_queues[j]);
 769                         return diag;
 770                 }
 771         }
 772
 773         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 774                 struct fm10k_tx_queue *txq;
 775                 txq = dev->data->tx_queues[i];
 776
 777                 if (txq->tx_deferred_start)
 778                         continue;
 779                 diag = fm10k_dev_tx_queue_start(dev, i);
 780                 if (diag != 0) {
 781                         int j;
 782                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 783                                 rx_queue_clean(dev->data->rx_queues[j]);
 784                         return diag;
 785                 }
 786         }
 787
 788         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID) {
 789                 fm10k_mbx_lock(hw);
 790                 /* Update default vlan */
 791                 hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
 792
 793                 /* Add default mac/vlan filter to PF/Switch manager */
 794                 hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
 795                                 hw->mac.default_vid, true, 0);
 796                 fm10k_mbx_unlock(hw);
 797         }
 798
 799         return 0;
 800 }
 801
 802 static void
 803 fm10k_dev_stop(struct rte_eth_dev *dev)
 804 {
 805         int i;
 806
 807         PMD_INIT_FUNC_TRACE();
 808
 809         for (i = 0; i < dev->data->nb_tx_queues; i++)
 810                 fm10k_dev_tx_queue_stop(dev, i);
 811
 812         for (i = 0; i < dev->data->nb_rx_queues; i++)
 813                 fm10k_dev_rx_queue_stop(dev, i);
 814 }
 815
 816 static void
 817 fm10k_dev_close(struct rte_eth_dev *dev)
 818 {
 819         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 820
 821         PMD_INIT_FUNC_TRACE();
 822
 823         /* Stop mailbox service first */
 824         fm10k_close_mbx_service(hw);
 825         fm10k_dev_stop(dev);
 826         fm10k_stop_hw(hw);
 827 }
 828
 829 static int
 830 fm10k_link_update(struct rte_eth_dev *dev,
 831         __rte_unused int wait_to_complete)
 832 {
 833         PMD_INIT_FUNC_TRACE();
 834
 835         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 836          * x8 PCIe interface. For now, we leave the speed undefined since there
 837          * is no 50Gbps Ethernet. */
 838         dev->data->dev_link.link_speed  = 0;
 839         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 840         dev->data->dev_link.link_status = 1;
 841
 842         return 0;
 843 }
 844
 845 static void
 846 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 847 {
 848         uint64_t ipackets, opackets, ibytes, obytes;
 849         struct fm10k_hw *hw =
 850                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 851         struct fm10k_hw_stats *hw_stats =
 852                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 853         int i;
 854
 855         PMD_INIT_FUNC_TRACE();
 856
 857         fm10k_update_hw_stats(hw, hw_stats);
 858
 859         ipackets = opackets = ibytes = obytes = 0;
 860         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 861                 (i < hw->mac.max_queues); ++i) {
 862                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 863                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 864                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 865                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 866                 ipackets += stats->q_ipackets[i];
 867                 opackets += stats->q_opackets[i];
 868                 ibytes   += stats->q_ibytes[i];
 869                 obytes   += stats->q_obytes[i];
 870         }
 871         stats->ipackets = ipackets;
 872         stats->opackets = opackets;
 873         stats->ibytes = ibytes;
 874         stats->obytes = obytes;
 875 }
 876
 877 static void
 878 fm10k_stats_reset(struct rte_eth_dev *dev)
 879 {
 880         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 881         struct fm10k_hw_stats *hw_stats =
 882                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 883
 884         PMD_INIT_FUNC_TRACE();
 885
 886         memset(hw_stats, 0, sizeof(*hw_stats));
 887         fm10k_rebind_hw_stats(hw, hw_stats);
 888 }
 889
 890 static void
 891 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 892         struct rte_eth_dev_info *dev_info)
 893 {
 894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 895
 896         PMD_INIT_FUNC_TRACE();
 897
 898         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 899         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 900         dev_info->max_rx_queues      = hw->mac.max_queues;
 901         dev_info->max_tx_queues      = hw->mac.max_queues;
 902         dev_info->max_mac_addrs      = 1;
 903         dev_info->max_hash_mac_addrs = 0;
 904         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 905         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 906         dev_info->rx_offload_capa =
 907                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 908                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 909                 DEV_RX_OFFLOAD_TCP_CKSUM;
 910         dev_info->tx_offload_capa    = 0;
 911         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 912
 913         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 914                 .rx_thresh = {
 915                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 916                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 917                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 918                 },
 919                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 920                 .rx_drop_en = 0,
 921         };
 922
 923         dev_info->default_txconf = (struct rte_eth_txconf) {
 924                 .tx_thresh = {
 925                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 926                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 927                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 928                 },
 929                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 930                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 931                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 932                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 933         };
 934
 935 }
 936
 937 static int
 938 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 939 {
 940         s32 result;
 941         uint32_t vid_idx, vid_bit, mac_index;
 942         struct fm10k_hw *hw;
 943         struct fm10k_macvlan_filter_info *macvlan;
 944         struct rte_eth_dev_data *data = dev->data;
 945
 946         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 947         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 948
 949         /* @todo - add support for the VF */
 950         if (hw->mac.type != fm10k_mac_pf) {
 951                 PMD_INIT_LOG(ERR, "VLAN filter not available on VF");
 952                 return -ENOTSUP;
 953         }
 954
 955         if (vlan_id > ETH_VLAN_ID_MAX) {
 956                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
 957                 return (-EINVAL);
 958         }
 959
 960         vid_idx = FM10K_VFTA_IDX(vlan_id);
 961         vid_bit = FM10K_VFTA_BIT(vlan_id);
 962         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
 963         if (on && (macvlan->vfta[vid_idx] & vid_bit))
 964                 return 0;
 965         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
 966         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
 967                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
 968                         "in the VLAN filter table");
 969                 return (-EINVAL);
 970         }
 971
 972         fm10k_mbx_lock(hw);
 973         result = fm10k_update_vlan(hw, vlan_id, 0, on);
 974         fm10k_mbx_unlock(hw);
 975         if (result != FM10K_SUCCESS) {
 976                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
 977                 return (-EIO);
 978         }
 979
 980         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
 981                         (result == FM10K_SUCCESS); mac_index++) {
 982                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
 983                         continue;
 984                 fm10k_mbx_lock(hw);
 985                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
 986                         data->mac_addrs[mac_index].addr_bytes,
 987                         vlan_id, on, 0);
 988                 fm10k_mbx_unlock(hw);
 989         }
 990         if (result != FM10K_SUCCESS) {
 991                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
 992                 return (-EIO);
 993         }
 994
 995         if (on) {
 996                 macvlan->vlan_num++;
 997                 macvlan->vfta[vid_idx] |= vid_bit;
 998         } else {
 999                 macvlan->vlan_num--;
1000                 macvlan->vfta[vid_idx] &= ~vid_bit;
1001         }
1002         return 0;
1003 }
1004
1005 static inline int
1006 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1007 {
1008         if ((request < min) || (request > max) || ((request % mult) != 0))
1009                 return -1;
1010         else
1011                 return 0;
1012 }
1013
1014 /*
1015  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1016  * the physical address is required. If the memzone is already created, then
1017  * this function returns a pointer to the existing memzone.
1018  */
1019 static inline const struct rte_memzone *
1020 allocate_hw_ring(const char *driver_name, const char *ring_name,
1021         uint8_t port_id, uint16_t queue_id, int socket_id,
1022         uint32_t size, uint32_t align)
1023 {
1024         char name[RTE_MEMZONE_NAMESIZE];
1025         const struct rte_memzone *mz;
1026
1027         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1028                  driver_name, ring_name, port_id, queue_id, socket_id);
1029
1030         /* return the memzone if it already exists */
1031         mz = rte_memzone_lookup(name);
1032         if (mz)
1033                 return mz;
1034
1035 #ifdef RTE_LIBRTE_XEN_DOM0
1036         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1037                                            RTE_PGSIZE_2M);
1038 #else
1039         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1040 #endif
1041 }
1042
1043 static inline int
1044 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1045 {
1046         if ((request < min) || (request > max) || ((div % request) != 0))
1047                 return -1;
1048         else
1049                 return 0;
1050 }
1051
1052 static inline int
1053 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1054 {
1055         uint16_t rx_free_thresh;
1056
1057         if (conf->rx_free_thresh == 0)
1058                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1059         else
1060                 rx_free_thresh = conf->rx_free_thresh;
1061
1062         /* make sure the requested threshold satisfies the constraints */
1063         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1064                         FM10K_RX_FREE_THRESH_MAX(q),
1065                         FM10K_RX_FREE_THRESH_DIV(q),
1066                         rx_free_thresh)) {
1067                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1068                         "less than or equal to %u, "
1069                         "greater than or equal to %u, "
1070                         "and a divisor of %u",
1071                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1072                         FM10K_RX_FREE_THRESH_MIN(q),
1073                         FM10K_RX_FREE_THRESH_DIV(q));
1074                 return (-EINVAL);
1075         }
1076
1077         q->alloc_thresh = rx_free_thresh;
1078         q->drop_en = conf->rx_drop_en;
1079         q->rx_deferred_start = conf->rx_deferred_start;
1080
1081         return 0;
1082 }
1083
1084 /*
1085  * Hardware requires specific alignment for Rx packet buffers. At
1086  * least one of the following two conditions must be satisfied.
1087  *  1. Address is 512B aligned
1088  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1089  *
1090  * As such, the driver may need to adjust the DMA address within the
1091  * buffer by up to 512B.
1092  *
1093  * return 1 if the element size is valid, otherwise return 0.
1094  */
1095 static int
1096 mempool_element_size_valid(struct rte_mempool *mp)
1097 {
1098         uint32_t min_size;
1099
1100         /* elt_size includes mbuf header and headroom */
1101         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1102                         RTE_PKTMBUF_HEADROOM;
1103
1104         /* account for up to 512B of alignment */
1105         min_size -= FM10K_RX_DATABUF_ALIGN;
1106
1107         /* sanity check for overflow */
1108         if (min_size > mp->elt_size)
1109                 return 0;
1110
1111         /* size is valid */
1112         return 1;
1113 }
1114
1115 static int
1116 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1117         uint16_t nb_desc, unsigned int socket_id,
1118         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1119 {
1120         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1121         struct fm10k_rx_queue *q;
1122         const struct rte_memzone *mz;
1123
1124         PMD_INIT_FUNC_TRACE();
1125
1126         /* make sure the mempool element size can account for alignment. */
1127         if (!mempool_element_size_valid(mp)) {
1128                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1129                 return (-EINVAL);
1130         }
1131
1132         /* make sure a valid number of descriptors have been requested */
1133         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1134                                 FM10K_MULT_RX_DESC, nb_desc)) {
1135                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1136                         "less than or equal to %"PRIu32", "
1137                         "greater than or equal to %u, "
1138                         "and a multiple of %u",
1139                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1140                         FM10K_MULT_RX_DESC);
1141                 return (-EINVAL);
1142         }
1143
1144         /*
1145          * if this queue existed already, free the associated memory. The
1146          * queue cannot be reused in case we need to allocate memory on
1147          * different socket than was previously used.
1148          */
1149         if (dev->data->rx_queues[queue_id] != NULL) {
1150                 rx_queue_free(dev->data->rx_queues[queue_id]);
1151                 dev->data->rx_queues[queue_id] = NULL;
1152         }
1153
1154         /* allocate memory for the queue structure */
1155         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1156                                 socket_id);
1157         if (q == NULL) {
1158                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1159                 return (-ENOMEM);
1160         }
1161
1162         /* setup queue */
1163         q->mp = mp;
1164         q->nb_desc = nb_desc;
1165         q->port_id = dev->data->port_id;
1166         q->queue_id = queue_id;
1167         q->tail_ptr = (volatile uint32_t *)
1168                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1169         if (handle_rxconf(q, conf))
1170                 return (-EINVAL);
1171
1172         /* allocate memory for the software ring */
1173         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1174                                         nb_desc * sizeof(struct rte_mbuf *),
1175                                         RTE_CACHE_LINE_SIZE, socket_id);
1176         if (q->sw_ring == NULL) {
1177                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1178                 rte_free(q);
1179                 return (-ENOMEM);
1180         }
1181
1182         /*
1183          * allocate memory for the hardware descriptor ring. A memzone large
1184          * enough to hold the maximum ring size is requested to allow for
1185          * resizing in later calls to the queue setup function.
1186          */
1187         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1188                                 dev->data->port_id, queue_id, socket_id,
1189                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1190         if (mz == NULL) {
1191                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1192                 rte_free(q->sw_ring);
1193                 rte_free(q);
1194                 return (-ENOMEM);
1195         }
1196         q->hw_ring = mz->addr;
1197         q->hw_ring_phys_addr = mz->phys_addr;
1198
1199         dev->data->rx_queues[queue_id] = q;
1200         return 0;
1201 }
1202
1203 static void
1204 fm10k_rx_queue_release(void *queue)
1205 {
1206         PMD_INIT_FUNC_TRACE();
1207
1208         rx_queue_free(queue);
1209 }
1210
1211 static inline int
1212 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1213 {
1214         uint16_t tx_free_thresh;
1215         uint16_t tx_rs_thresh;
1216
1217         /* constraint MACROs require that tx_free_thresh is configured
1218          * before tx_rs_thresh */
1219         if (conf->tx_free_thresh == 0)
1220                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1221         else
1222                 tx_free_thresh = conf->tx_free_thresh;
1223
1224         /* make sure the requested threshold satisfies the constraints */
1225         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1226                         FM10K_TX_FREE_THRESH_MAX(q),
1227                         FM10K_TX_FREE_THRESH_DIV(q),
1228                         tx_free_thresh)) {
1229                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1230                         "less than or equal to %u, "
1231                         "greater than or equal to %u, "
1232                         "and a divisor of %u",
1233                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1234                         FM10K_TX_FREE_THRESH_MIN(q),
1235                         FM10K_TX_FREE_THRESH_DIV(q));
1236                 return (-EINVAL);
1237         }
1238
1239         q->free_thresh = tx_free_thresh;
1240
1241         if (conf->tx_rs_thresh == 0)
1242                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1243         else
1244                 tx_rs_thresh = conf->tx_rs_thresh;
1245
1246         q->tx_deferred_start = conf->tx_deferred_start;
1247
1248         /* make sure the requested threshold satisfies the constraints */
1249         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1250                         FM10K_TX_RS_THRESH_MAX(q),
1251                         FM10K_TX_RS_THRESH_DIV(q),
1252                         tx_rs_thresh)) {
1253                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1254                         "less than or equal to %u, "
1255                         "greater than or equal to %u, "
1256                         "and a divisor of %u",
1257                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1258                         FM10K_TX_RS_THRESH_MIN(q),
1259                         FM10K_TX_RS_THRESH_DIV(q));
1260                 return (-EINVAL);
1261         }
1262
1263         q->rs_thresh = tx_rs_thresh;
1264
1265         return 0;
1266 }
1267
1268 static int
1269 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1270         uint16_t nb_desc, unsigned int socket_id,
1271         const struct rte_eth_txconf *conf)
1272 {
1273         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1274         struct fm10k_tx_queue *q;
1275         const struct rte_memzone *mz;
1276
1277         PMD_INIT_FUNC_TRACE();
1278
1279         /* make sure a valid number of descriptors have been requested */
1280         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1281                                 FM10K_MULT_TX_DESC, nb_desc)) {
1282                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1283                         "less than or equal to %"PRIu32", "
1284                         "greater than or equal to %u, "
1285                         "and a multiple of %u",
1286                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1287                         FM10K_MULT_TX_DESC);
1288                 return (-EINVAL);
1289         }
1290
1291         /*
1292          * if this queue existed already, free the associated memory. The
1293          * queue cannot be reused in case we need to allocate memory on
1294          * different socket than was previously used.
1295          */
1296         if (dev->data->tx_queues[queue_id] != NULL) {
1297                 tx_queue_free(dev->data->tx_queues[queue_id]);
1298                 dev->data->tx_queues[queue_id] = NULL;
1299         }
1300
1301         /* allocate memory for the queue structure */
1302         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1303                                 socket_id);
1304         if (q == NULL) {
1305                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1306                 return (-ENOMEM);
1307         }
1308
1309         /* setup queue */
1310         q->nb_desc = nb_desc;
1311         q->port_id = dev->data->port_id;
1312         q->queue_id = queue_id;
1313         q->tail_ptr = (volatile uint32_t *)
1314                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1315         if (handle_txconf(q, conf))
1316                 return (-EINVAL);
1317
1318         /* allocate memory for the software ring */
1319         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1320                                         nb_desc * sizeof(struct rte_mbuf *),
1321                                         RTE_CACHE_LINE_SIZE, socket_id);
1322         if (q->sw_ring == NULL) {
1323                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1324                 rte_free(q);
1325                 return (-ENOMEM);
1326         }
1327
1328         /*
1329          * allocate memory for the hardware descriptor ring. A memzone large
1330          * enough to hold the maximum ring size is requested to allow for
1331          * resizing in later calls to the queue setup function.
1332          */
1333         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1334                                 dev->data->port_id, queue_id, socket_id,
1335                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1336         if (mz == NULL) {
1337                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1338                 rte_free(q->sw_ring);
1339                 rte_free(q);
1340                 return (-ENOMEM);
1341         }
1342         q->hw_ring = mz->addr;
1343         q->hw_ring_phys_addr = mz->phys_addr;
1344
1345         /*
1346          * allocate memory for the RS bit tracker. Enough slots to hold the
1347          * descriptor index for each RS bit needing to be set are required.
1348          */
1349         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1350                                 ((nb_desc + 1) / q->rs_thresh) *
1351                                 sizeof(uint16_t),
1352                                 RTE_CACHE_LINE_SIZE, socket_id);
1353         if (q->rs_tracker.list == NULL) {
1354                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1355                 rte_free(q->sw_ring);
1356                 rte_free(q);
1357                 return (-ENOMEM);
1358         }
1359
1360         dev->data->tx_queues[queue_id] = q;
1361         return 0;
1362 }
1363
1364 static void
1365 fm10k_tx_queue_release(void *queue)
1366 {
1367         PMD_INIT_FUNC_TRACE();
1368
1369         tx_queue_free(queue);
1370 }
1371
1372 static int
1373 fm10k_reta_update(struct rte_eth_dev *dev,
1374                         struct rte_eth_rss_reta_entry64 *reta_conf,
1375                         uint16_t reta_size)
1376 {
1377         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1378         uint16_t i, j, idx, shift;
1379         uint8_t mask;
1380         uint32_t reta;
1381
1382         PMD_INIT_FUNC_TRACE();
1383
1384         if (reta_size > FM10K_MAX_RSS_INDICES) {
1385                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1386                         "(%d) doesn't match the number hardware can supported "
1387                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1388                 return -EINVAL;
1389         }
1390
1391         /*
1392          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1393          * 128-entries in 32 registers
1394          */
1395         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1396                 idx = i / RTE_RETA_GROUP_SIZE;
1397                 shift = i % RTE_RETA_GROUP_SIZE;
1398                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1399                                 BIT_MASK_PER_UINT32);
1400                 if (mask == 0)
1401                         continue;
1402
1403                 reta = 0;
1404                 if (mask != BIT_MASK_PER_UINT32)
1405                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1406
1407                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1408                         if (mask & (0x1 << j)) {
1409                                 if (mask != 0xF)
1410                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1411                                 reta |= reta_conf[idx].reta[shift + j] <<
1412                                                 (CHAR_BIT * j);
1413                         }
1414                 }
1415                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1416         }
1417
1418         return 0;
1419 }
1420
1421 static int
1422 fm10k_reta_query(struct rte_eth_dev *dev,
1423                         struct rte_eth_rss_reta_entry64 *reta_conf,
1424                         uint16_t reta_size)
1425 {
1426         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1427         uint16_t i, j, idx, shift;
1428         uint8_t mask;
1429         uint32_t reta;
1430
1431         PMD_INIT_FUNC_TRACE();
1432
1433         if (reta_size < FM10K_MAX_RSS_INDICES) {
1434                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1435                         "(%d) doesn't match the number hardware can supported "
1436                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1437                 return -EINVAL;
1438         }
1439
1440         /*
1441          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1442          * 128-entries in 32 registers
1443          */
1444         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1445                 idx = i / RTE_RETA_GROUP_SIZE;
1446                 shift = i % RTE_RETA_GROUP_SIZE;
1447                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1448                                 BIT_MASK_PER_UINT32);
1449                 if (mask == 0)
1450                         continue;
1451
1452                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1453                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1454                         if (mask & (0x1 << j))
1455                                 reta_conf[idx].reta[shift + j] = ((reta >>
1456                                         CHAR_BIT * j) & UINT8_MAX);
1457                 }
1458         }
1459
1460         return 0;
1461 }
1462
1463 static int
1464 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1465         struct rte_eth_rss_conf *rss_conf)
1466 {
1467         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1468         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1469         uint32_t mrqc;
1470         uint64_t hf = rss_conf->rss_hf;
1471         int i;
1472
1473         PMD_INIT_FUNC_TRACE();
1474
1475         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1476                 FM10K_RSSRK_ENTRIES_PER_REG)
1477                 return -EINVAL;
1478
1479         if (hf == 0)
1480                 return -EINVAL;
1481
1482         mrqc = 0;
1483         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1484         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1485         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1486         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1487         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1488         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1489         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1490         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1491         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1492
1493         /* If the mapping doesn't fit any supported, return */
1494         if (mrqc == 0)
1495                 return -EINVAL;
1496
1497         if (key != NULL)
1498                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1499                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1500
1501         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1502
1503         return 0;
1504 }
1505
1506 static int
1507 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1508         struct rte_eth_rss_conf *rss_conf)
1509 {
1510         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1511         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1512         uint32_t mrqc;
1513         uint64_t hf;
1514         int i;
1515
1516         PMD_INIT_FUNC_TRACE();
1517
1518         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1519                                 FM10K_RSSRK_ENTRIES_PER_REG)
1520                 return -EINVAL;
1521
1522         if (key != NULL)
1523                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1524                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1525
1526         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1527         hf = 0;
1528         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1529         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1530         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1531         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1532         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1533         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1534         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1535         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1536         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1537
1538         rss_conf->rss_hf = hf;
1539
1540         return 0;
1541 }
1542
1543 static void
1544 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1545 {
1546         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1547         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1548
1549         /* Bind all local non-queue interrupt to vector 0 */
1550         int_map |= 0;
1551
1552         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1553         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1554         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1555         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1556         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1557         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1558
1559         /* Enable misc causes */
1560         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1561                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1562                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1563                                 FM10K_EIMR_ENABLE(MAILBOX) |
1564                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1565                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1566                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1567                                 FM10K_EIMR_ENABLE(VFLR));
1568
1569         /* Enable ITR 0 */
1570         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1571                                         FM10K_ITR_MASK_CLEAR);
1572         FM10K_WRITE_FLUSH(hw);
1573 }
1574
1575 static void
1576 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1577 {
1578         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1579         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1580
1581         /* Bind all local non-queue interrupt to vector 0 */
1582         int_map |= 0;
1583
1584         /* Only INT 0 available, other 15 are reserved. */
1585         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1586
1587         /* Enable ITR 0 */
1588         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1589                                         FM10K_ITR_MASK_CLEAR);
1590         FM10K_WRITE_FLUSH(hw);
1591 }
1592
1593 static int
1594 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1595 {
1596         struct fm10k_fault fault;
1597         int err;
1598         const char *estr = "Unknown error";
1599
1600         /* Process PCA fault */
1601         if (eicr & FM10K_EIMR_PCA_FAULT) {
1602                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1603                 if (err)
1604                         goto error;
1605                 switch (fault.type) {
1606                 case PCA_NO_FAULT:
1607                         estr = "PCA_NO_FAULT"; break;
1608                 case PCA_UNMAPPED_ADDR:
1609                         estr = "PCA_UNMAPPED_ADDR"; break;
1610                 case PCA_BAD_QACCESS_PF:
1611                         estr = "PCA_BAD_QACCESS_PF"; break;
1612                 case PCA_BAD_QACCESS_VF:
1613                         estr = "PCA_BAD_QACCESS_VF"; break;
1614                 case PCA_MALICIOUS_REQ:
1615                         estr = "PCA_MALICIOUS_REQ"; break;
1616                 case PCA_POISONED_TLP:
1617                         estr = "PCA_POISONED_TLP"; break;
1618                 case PCA_TLP_ABORT:
1619                         estr = "PCA_TLP_ABORT"; break;
1620                 default:
1621                         goto error;
1622                 }
1623                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1624                         estr, fault.func ? "VF" : "PF", fault.func,
1625                         fault.address, fault.specinfo);
1626         }
1627
1628         /* Process THI fault */
1629         if (eicr & FM10K_EIMR_THI_FAULT) {
1630                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1631                 if (err)
1632                         goto error;
1633                 switch (fault.type) {
1634                 case THI_NO_FAULT:
1635                         estr = "THI_NO_FAULT"; break;
1636                 case THI_MAL_DIS_Q_FAULT:
1637                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1638                 default:
1639                         goto error;
1640                 }
1641                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1642                         estr, fault.func ? "VF" : "PF", fault.func,
1643                         fault.address, fault.specinfo);
1644         }
1645
1646         /* Process FUM fault */
1647         if (eicr & FM10K_EIMR_FUM_FAULT) {
1648                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1649                 if (err)
1650                         goto error;
1651                 switch (fault.type) {
1652                 case FUM_NO_FAULT:
1653                         estr = "FUM_NO_FAULT"; break;
1654                 case FUM_UNMAPPED_ADDR:
1655                         estr = "FUM_UNMAPPED_ADDR"; break;
1656                 case FUM_POISONED_TLP:
1657                         estr = "FUM_POISONED_TLP"; break;
1658                 case FUM_BAD_VF_QACCESS:
1659                         estr = "FUM_BAD_VF_QACCESS"; break;
1660                 case FUM_ADD_DECODE_ERR:
1661                         estr = "FUM_ADD_DECODE_ERR"; break;
1662                 case FUM_RO_ERROR:
1663                         estr = "FUM_RO_ERROR"; break;
1664                 case FUM_QPRC_CRC_ERROR:
1665                         estr = "FUM_QPRC_CRC_ERROR"; break;
1666                 case FUM_CSR_TIMEOUT:
1667                         estr = "FUM_CSR_TIMEOUT"; break;
1668                 case FUM_INVALID_TYPE:
1669                         estr = "FUM_INVALID_TYPE"; break;
1670                 case FUM_INVALID_LENGTH:
1671                         estr = "FUM_INVALID_LENGTH"; break;
1672                 case FUM_INVALID_BE:
1673                         estr = "FUM_INVALID_BE"; break;
1674                 case FUM_INVALID_ALIGN:
1675                         estr = "FUM_INVALID_ALIGN"; break;
1676                 default:
1677                         goto error;
1678                 }
1679                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1680                         estr, fault.func ? "VF" : "PF", fault.func,
1681                         fault.address, fault.specinfo);
1682         }
1683
1684         if (estr)
1685                 return 0;
1686         return 0;
1687 error:
1688         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1689         return err;
1690 }
1691
1692 /**
1693  * PF interrupt handler triggered by NIC for handling specific interrupt.
1694  *
1695  * @param handle
1696  *  Pointer to interrupt handle.
1697  * @param param
1698  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1699  *
1700  * @return
1701  *  void
1702  */
1703 static void
1704 fm10k_dev_interrupt_handler_pf(
1705                         __rte_unused struct rte_intr_handle *handle,
1706                         void *param)
1707 {
1708         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1709         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1710         uint32_t cause, status;
1711
1712         if (hw->mac.type != fm10k_mac_pf)
1713                 return;
1714
1715         cause = FM10K_READ_REG(hw, FM10K_EICR);
1716
1717         /* Handle PCI fault cases */
1718         if (cause & FM10K_EICR_FAULT_MASK) {
1719                 PMD_INIT_LOG(ERR, "INT: find fault!");
1720                 fm10k_dev_handle_fault(hw, cause);
1721         }
1722
1723         /* Handle switch up/down */
1724         if (cause & FM10K_EICR_SWITCHNOTREADY)
1725                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1726
1727         if (cause & FM10K_EICR_SWITCHREADY)
1728                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1729
1730         /* Handle mailbox message */
1731         fm10k_mbx_lock(hw);
1732         hw->mbx.ops.process(hw, &hw->mbx);
1733         fm10k_mbx_unlock(hw);
1734
1735         /* Handle SRAM error */
1736         if (cause & FM10K_EICR_SRAMERROR) {
1737                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1738
1739                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1740                 /* Write to clear pending bits */
1741                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1742
1743                 /* Todo: print out error message after shared code  updates */
1744         }
1745
1746         /* Clear these 3 events if having any */
1747         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1748                  FM10K_EICR_SWITCHREADY;
1749         if (cause)
1750                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1751
1752         /* Re-enable interrupt from device side */
1753         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1754                                         FM10K_ITR_MASK_CLEAR);
1755         /* Re-enable interrupt from host side */
1756         rte_intr_enable(&(dev->pci_dev->intr_handle));
1757 }
1758
1759 /**
1760  * VF interrupt handler triggered by NIC for handling specific interrupt.
1761  *
1762  * @param handle
1763  *  Pointer to interrupt handle.
1764  * @param param
1765  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1766  *
1767  * @return
1768  *  void
1769  */
1770 static void
1771 fm10k_dev_interrupt_handler_vf(
1772                         __rte_unused struct rte_intr_handle *handle,
1773                         void *param)
1774 {
1775         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1776         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1777
1778         if (hw->mac.type != fm10k_mac_vf)
1779                 return;
1780
1781         /* Handle mailbox message if lock is acquired */
1782         fm10k_mbx_lock(hw);
1783         hw->mbx.ops.process(hw, &hw->mbx);
1784         fm10k_mbx_unlock(hw);
1785
1786         /* Re-enable interrupt from device side */
1787         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1788                                         FM10K_ITR_MASK_CLEAR);
1789         /* Re-enable interrupt from host side */
1790         rte_intr_enable(&(dev->pci_dev->intr_handle));
1791 }
1792
1793 /* Mailbox message handler in VF */
1794 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1795         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1796         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1797         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1798         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1799 };
1800
1801 /* Mailbox message handler in PF */
1802 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1803         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1804         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1805         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1806         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1807         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1808         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1809         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1810 };
1811
1812 static int
1813 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1814 {
1815         int err;
1816
1817         /* Initialize mailbox lock */
1818         fm10k_mbx_initlock(hw);
1819
1820         /* Replace default message handler with new ones */
1821         if (hw->mac.type == fm10k_mac_pf)
1822                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1823         else
1824                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1825
1826         if (err) {
1827                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1828                                 err);
1829                 return err;
1830         }
1831         /* Connect to SM for PF device or PF for VF device */
1832         return hw->mbx.ops.connect(hw, &hw->mbx);
1833 }
1834
1835 static void
1836 fm10k_close_mbx_service(struct fm10k_hw *hw)
1837 {
1838         /* Disconnect from SM for PF device or PF for VF device */
1839         hw->mbx.ops.disconnect(hw, &hw->mbx);
1840 }
1841
1842 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1843         .dev_configure          = fm10k_dev_configure,
1844         .dev_start              = fm10k_dev_start,
1845         .dev_stop               = fm10k_dev_stop,
1846         .dev_close              = fm10k_dev_close,
1847         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
1848         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
1849         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
1850         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
1851         .stats_get              = fm10k_stats_get,
1852         .stats_reset            = fm10k_stats_reset,
1853         .link_update            = fm10k_link_update,
1854         .dev_infos_get          = fm10k_dev_infos_get,
1855         .vlan_filter_set        = fm10k_vlan_filter_set,
1856         .rx_queue_start         = fm10k_dev_rx_queue_start,
1857         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1858         .tx_queue_start         = fm10k_dev_tx_queue_start,
1859         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1860         .rx_queue_setup         = fm10k_rx_queue_setup,
1861         .rx_queue_release       = fm10k_rx_queue_release,
1862         .tx_queue_setup         = fm10k_tx_queue_setup,
1863         .tx_queue_release       = fm10k_tx_queue_release,
1864         .reta_update            = fm10k_reta_update,
1865         .reta_query             = fm10k_reta_query,
1866         .rss_hash_update        = fm10k_rss_hash_update,
1867         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1868 };
1869
1870 static int
1871 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1872 {
1873         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1874         int diag;
1875         struct fm10k_macvlan_filter_info *macvlan;
1876
1877         PMD_INIT_FUNC_TRACE();
1878
1879         dev->dev_ops = &fm10k_eth_dev_ops;
1880         dev->rx_pkt_burst = &fm10k_recv_pkts;
1881         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1882
1883         if (dev->data->scattered_rx)
1884                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1885
1886         /* only initialize in the primary process */
1887         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1888                 return 0;
1889
1890         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1891         memset(macvlan, 0, sizeof(*macvlan));
1892         /* Vendor and Device ID need to be set before init of shared code */
1893         memset(hw, 0, sizeof(*hw));
1894         hw->device_id = dev->pci_dev->id.device_id;
1895         hw->vendor_id = dev->pci_dev->id.vendor_id;
1896         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1897         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1898         hw->revision_id = 0;
1899         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1900         if (hw->hw_addr == NULL) {
1901                 PMD_INIT_LOG(ERR, "Bad mem resource."
1902                         " Try to blacklist unused devices.");
1903                 return -EIO;
1904         }
1905
1906         /* Store fm10k_adapter pointer */
1907         hw->back = dev->data->dev_private;
1908
1909         /* Initialize the shared code */
1910         diag = fm10k_init_shared_code(hw);
1911         if (diag != FM10K_SUCCESS) {
1912                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1913                 return -EIO;
1914         }
1915
1916         /*
1917          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1918          * there is no way to get link status without reading BAR4.  Until this
1919          * works, assume we have maximum bandwidth.
1920          * @todo - fix bus info
1921          */
1922         hw->bus_caps.speed = fm10k_bus_speed_8000;
1923         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1924         hw->bus_caps.payload = fm10k_bus_payload_512;
1925         hw->bus.speed = fm10k_bus_speed_8000;
1926         hw->bus.width = fm10k_bus_width_pcie_x8;
1927         hw->bus.payload = fm10k_bus_payload_256;
1928
1929         /* Initialize the hw */
1930         diag = fm10k_init_hw(hw);
1931         if (diag != FM10K_SUCCESS) {
1932                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1933                 return -EIO;
1934         }
1935
1936         /* Initialize MAC address(es) */
1937         dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1938         if (dev->data->mac_addrs == NULL) {
1939                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1940                 return -ENOMEM;
1941         }
1942
1943         diag = fm10k_read_mac_addr(hw);
1944
1945         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1946                         &dev->data->mac_addrs[0]);
1947
1948         if (diag != FM10K_SUCCESS ||
1949                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
1950
1951                 /* Generate a random addr */
1952                 eth_random_addr(hw->mac.addr);
1953                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1954                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1955                 &dev->data->mac_addrs[0]);
1956         }
1957
1958         /* Reset the hw statistics */
1959         fm10k_stats_reset(dev);
1960
1961         /* Reset the hw */
1962         diag = fm10k_reset_hw(hw);
1963         if (diag != FM10K_SUCCESS) {
1964                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1965                 return -EIO;
1966         }
1967
1968         /* Setup mailbox service */
1969         diag = fm10k_setup_mbx_service(hw);
1970         if (diag != FM10K_SUCCESS) {
1971                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1972                 return -EIO;
1973         }
1974
1975         /*PF/VF has different interrupt handling mechanism */
1976         if (hw->mac.type == fm10k_mac_pf) {
1977                 /* register callback func to eal lib */
1978                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1979                         fm10k_dev_interrupt_handler_pf, (void *)dev);
1980
1981                 /* enable MISC interrupt */
1982                 fm10k_dev_enable_intr_pf(dev);
1983         } else { /* VF */
1984                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1985                         fm10k_dev_interrupt_handler_vf, (void *)dev);
1986
1987                 fm10k_dev_enable_intr_vf(dev);
1988         }
1989
1990         /* Enable uio intr after callback registered */
1991         rte_intr_enable(&(dev->pci_dev->intr_handle));
1992
1993         hw->mac.ops.update_int_moderator(hw);
1994
1995         /* Make sure Switch Manager is ready before going forward. */
1996         if (hw->mac.type == fm10k_mac_pf) {
1997                 int switch_ready = 0;
1998                 int i;
1999
2000                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2001                         fm10k_mbx_lock(hw);
2002                         hw->mac.ops.get_host_state(hw, &switch_ready);
2003                         fm10k_mbx_unlock(hw);
2004                         if (switch_ready)
2005                                 break;
2006                         /* Delay some time to acquire async LPORT_MAP info. */
2007                         rte_delay_us(WAIT_SWITCH_MSG_US);
2008                 }
2009
2010                 if (switch_ready == 0) {
2011                         PMD_INIT_LOG(ERR, "switch is not ready");
2012                         return -1;
2013                 }
2014         }
2015
2016         /*
2017          * Below function will trigger operations on mailbox, acquire lock to
2018          * avoid race condition from interrupt handler. Operations on mailbox
2019          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2020          * will handle and generate an interrupt to our side. Then,  FIFO in
2021          * mailbox will be touched.
2022          */
2023         fm10k_mbx_lock(hw);
2024         /* Enable port first */
2025         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2026
2027         /*
2028          * Add default mac. glort is assigned by SM for PF, while is
2029          * unused for VF. PF will assign correct glort for VF.
2030          */
2031         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
2032                                 0, 1, 0);
2033
2034         /* Set unicast mode by default. App can change to other mode in other
2035          * API func.
2036          */
2037         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2038                                         FM10K_XCAST_MODE_NONE);
2039
2040         fm10k_mbx_unlock(hw);
2041
2042
2043         return 0;
2044 }
2045
2046 /*
2047  * The set of PCI devices this driver supports. This driver will enable both PF
2048  * and SRIOV-VF devices.
2049  */
2050 static const struct rte_pci_id pci_id_fm10k_map[] = {
2051 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2052 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2053 #include "rte_pci_dev_ids.h"
2054         { .vendor_id = 0, /* sentinel */ },
2055 };
2056
2057 static struct eth_driver rte_pmd_fm10k = {
2058         .pci_drv = {
2059                 .name = "rte_pmd_fm10k",
2060                 .id_table = pci_id_fm10k_map,
2061                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
2062         },
2063         .eth_dev_init = eth_fm10k_dev_init,
2064         .dev_private_size = sizeof(struct fm10k_adapter),
2065 };
2066
2067 /*
2068  * Driver initialization routine.
2069  * Invoked once at EAL init time.
2070  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2071  */
2072 static int
2073 rte_pmd_fm10k_init(__rte_unused const char *name,
2074         __rte_unused const char *params)
2075 {
2076         PMD_INIT_FUNC_TRACE();
2077         rte_eth_driver_register(&rte_pmd_fm10k);
2078         return 0;
2079 }
2080
2081 static struct rte_driver rte_fm10k_driver = {
2082         .type = PMD_PDEV,
2083         .init = rte_pmd_fm10k_init,
2084 };
2085
2086 PMD_REGISTER_DRIVER(rte_fm10k_driver);