drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void
  65 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add);
  66
  67 static void
  68 fm10k_mbx_initlock(struct fm10k_hw *hw)
  69 {
  70         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  71 }
  72
  73 static void
  74 fm10k_mbx_lock(struct fm10k_hw *hw)
  75 {
  76         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  77                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  78 }
  79
  80 static void
  81 fm10k_mbx_unlock(struct fm10k_hw *hw)
  82 {
  83         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  84 }
  85
  86 /*
  87  * reset queue to initial state, allocate software buffers used when starting
  88  * device.
  89  * return 0 on success
  90  * return -ENOMEM if buffers cannot be allocated
  91  * return -EINVAL if buffers do not satisfy alignment condition
  92  */
  93 static inline int
  94 rx_queue_reset(struct fm10k_rx_queue *q)
  95 {
  96         uint64_t dma_addr;
  97         int i, diag;
  98         PMD_INIT_FUNC_TRACE();
  99
 100         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 101         if (diag != 0)
 102                 return -ENOMEM;
 103
 104         for (i = 0; i < q->nb_desc; ++i) {
 105                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 106                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 107                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 108                                                 q->nb_desc);
 109                         return -EINVAL;
 110                 }
 111                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 112                 q->hw_ring[i].q.pkt_addr = dma_addr;
 113                 q->hw_ring[i].q.hdr_addr = dma_addr;
 114         }
 115
 116         q->next_dd = 0;
 117         q->next_alloc = 0;
 118         q->next_trigger = q->alloc_thresh - 1;
 119         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 120         return 0;
 121 }
 122
 123 /*
 124  * clean queue, descriptor rings, free software buffers used when stopping
 125  * device.
 126  */
 127 static inline void
 128 rx_queue_clean(struct fm10k_rx_queue *q)
 129 {
 130         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 131         uint32_t i;
 132         PMD_INIT_FUNC_TRACE();
 133
 134         /* zero descriptor rings */
 135         for (i = 0; i < q->nb_desc; ++i)
 136                 q->hw_ring[i] = zero;
 137
 138         /* free software buffers */
 139         for (i = 0; i < q->nb_desc; ++i) {
 140                 if (q->sw_ring[i]) {
 141                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 142                         q->sw_ring[i] = NULL;
 143                 }
 144         }
 145 }
 146
 147 /*
 148  * free all queue memory used when releasing the queue (i.e. configure)
 149  */
 150 static inline void
 151 rx_queue_free(struct fm10k_rx_queue *q)
 152 {
 153         PMD_INIT_FUNC_TRACE();
 154         if (q) {
 155                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 156                 rx_queue_clean(q);
 157                 if (q->sw_ring) {
 158                         rte_free(q->sw_ring);
 159                         q->sw_ring = NULL;
 160                 }
 161                 rte_free(q);
 162                 q = NULL;
 163         }
 164 }
 165
 166 /*
 167  * disable RX queue, wait unitl HW finished necessary flush operation
 168  */
 169 static inline int
 170 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 171 {
 172         uint32_t reg, i;
 173
 174         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 175         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 176                         reg & ~FM10K_RXQCTL_ENABLE);
 177
 178         /* Wait 100us at most */
 179         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 180                 rte_delay_us(1);
 181                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 182                 if (!(reg & FM10K_RXQCTL_ENABLE))
 183                         break;
 184         }
 185
 186         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 187                 return -1;
 188
 189         return 0;
 190 }
 191
 192 /*
 193  * reset queue to initial state, allocate software buffers used when starting
 194  * device
 195  */
 196 static inline void
 197 tx_queue_reset(struct fm10k_tx_queue *q)
 198 {
 199         PMD_INIT_FUNC_TRACE();
 200         q->last_free = 0;
 201         q->next_free = 0;
 202         q->nb_used = 0;
 203         q->nb_free = q->nb_desc - 1;
 204         q->free_trigger = q->nb_free - q->free_thresh;
 205         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 206         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 207 }
 208
 209 /*
 210  * clean queue, descriptor rings, free software buffers used when stopping
 211  * device
 212  */
 213 static inline void
 214 tx_queue_clean(struct fm10k_tx_queue *q)
 215 {
 216         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 217         uint32_t i;
 218         PMD_INIT_FUNC_TRACE();
 219
 220         /* zero descriptor rings */
 221         for (i = 0; i < q->nb_desc; ++i)
 222                 q->hw_ring[i] = zero;
 223
 224         /* free software buffers */
 225         for (i = 0; i < q->nb_desc; ++i) {
 226                 if (q->sw_ring[i]) {
 227                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 228                         q->sw_ring[i] = NULL;
 229                 }
 230         }
 231 }
 232
 233 /*
 234  * free all queue memory used when releasing the queue (i.e. configure)
 235  */
 236 static inline void
 237 tx_queue_free(struct fm10k_tx_queue *q)
 238 {
 239         PMD_INIT_FUNC_TRACE();
 240         if (q) {
 241                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 242                 tx_queue_clean(q);
 243                 if (q->rs_tracker.list) {
 244                         rte_free(q->rs_tracker.list);
 245                         q->rs_tracker.list = NULL;
 246                 }
 247                 if (q->sw_ring) {
 248                         rte_free(q->sw_ring);
 249                         q->sw_ring = NULL;
 250                 }
 251                 rte_free(q);
 252                 q = NULL;
 253         }
 254 }
 255
 256 /*
 257  * disable TX queue, wait unitl HW finished necessary flush operation
 258  */
 259 static inline int
 260 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 261 {
 262         uint32_t reg, i;
 263
 264         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 265         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 266                         reg & ~FM10K_TXDCTL_ENABLE);
 267
 268         /* Wait 100us at most */
 269         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 270                 rte_delay_us(1);
 271                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 272                 if (!(reg & FM10K_TXDCTL_ENABLE))
 273                         break;
 274         }
 275
 276         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 277                 return -1;
 278
 279         return 0;
 280 }
 281
 282 static int
 283 fm10k_dev_configure(struct rte_eth_dev *dev)
 284 {
 285         PMD_INIT_FUNC_TRACE();
 286
 287         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 288                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 289
 290         return 0;
 291 }
 292
 293 static void
 294 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 295 {
 296         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 297         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 298         uint32_t mrqc, *key, i, reta, j;
 299         uint64_t hf;
 300
 301 #define RSS_KEY_SIZE 40
 302         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 303                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 304                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 305                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 306                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 307                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 308         };
 309
 310         if (dev->data->nb_rx_queues == 1 ||
 311             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 312             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 313                 return;
 314
 315         /* random key is rss_intel_key (default) or user provided (rss_key) */
 316         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 317                 key = (uint32_t *)rss_intel_key;
 318         else
 319                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 320
 321         /* Now fill our hash function seeds, 4 bytes at a time */
 322         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 323                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 324
 325         /*
 326          * Fill in redirection table
 327          * The byte-swap is needed because NIC registers are in
 328          * little-endian order.
 329          */
 330         reta = 0;
 331         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 332                 if (j == dev->data->nb_rx_queues)
 333                         j = 0;
 334                 reta = (reta << CHAR_BIT) | j;
 335                 if ((i & 3) == 3)
 336                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 337                                         rte_bswap32(reta));
 338         }
 339
 340         /*
 341          * Generate RSS hash based on packet types, TCP/UDP
 342          * port numbers and/or IPv4/v6 src and dst addresses
 343          */
 344         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 345         mrqc = 0;
 346         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 347         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 348         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 349         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 350         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 351         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 352         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 353         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 354         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 355
 356         if (mrqc == 0) {
 357                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 358                         "supported", hf);
 359                 return;
 360         }
 361
 362         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 363 }
 364
 365 static int
 366 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 367 {
 368         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 369         int i, ret;
 370         struct fm10k_tx_queue *txq;
 371         uint64_t base_addr;
 372         uint32_t size;
 373
 374         /* Disable TXINT to avoid possible interrupt */
 375         for (i = 0; i < hw->mac.max_queues; i++)
 376                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 377                                 3 << FM10K_TXINT_TIMER_SHIFT);
 378
 379         /* Setup TX queue */
 380         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 381                 txq = dev->data->tx_queues[i];
 382                 base_addr = txq->hw_ring_phys_addr;
 383                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 384
 385                 /* disable queue to avoid issues while updating state */
 386                 ret = tx_queue_disable(hw, i);
 387                 if (ret) {
 388                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 389                         return -1;
 390                 }
 391
 392                 /* set location and size for descriptor ring */
 393                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 394                                 base_addr & UINT64_LOWER_32BITS_MASK);
 395                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 396                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 397                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 398         }
 399         return 0;
 400 }
 401
 402 static int
 403 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 404 {
 405         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 406         int i, ret;
 407         struct fm10k_rx_queue *rxq;
 408         uint64_t base_addr;
 409         uint32_t size;
 410         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 411         uint16_t buf_size;
 412
 413         /* Disable RXINT to avoid possible interrupt */
 414         for (i = 0; i < hw->mac.max_queues; i++)
 415                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 416                                 3 << FM10K_RXINT_TIMER_SHIFT);
 417
 418         /* Setup RX queues */
 419         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 420                 rxq = dev->data->rx_queues[i];
 421                 base_addr = rxq->hw_ring_phys_addr;
 422                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 423
 424                 /* disable queue to avoid issues while updating state */
 425                 ret = rx_queue_disable(hw, i);
 426                 if (ret) {
 427                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 428                         return -1;
 429                 }
 430
 431                 /* Setup the Base and Length of the Rx Descriptor Ring */
 432                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 433                                 base_addr & UINT64_LOWER_32BITS_MASK);
 434                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 435                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 436                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 437
 438                 /* Configure the Rx buffer size for one buff without split */
 439                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 440                         RTE_PKTMBUF_HEADROOM);
 441                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 442                  * reserved for this purpose, and the worst case could be 511B.
 443                  * But SRR reg assumes all buffers have the same size. In order
 444                  * to fill the gap, we'll have to consider the worst case and
 445                  * assume 512B is reserved. If we don't do so, it's possible
 446                  * for HW to overwrite data to next mbuf.
 447                  */
 448                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 449
 450                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 451                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 452
 453                 /* It adds dual VLAN length for supporting dual VLAN */
 454                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 455                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 456                         dev->data->dev_conf.rxmode.enable_scatter) {
 457                         uint32_t reg;
 458                         dev->data->scattered_rx = 1;
 459                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 460                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 461                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 462                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 463                 }
 464
 465                 /* Enable drop on empty, it's RO for VF */
 466                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 467                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 468
 469                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 470                 FM10K_WRITE_FLUSH(hw);
 471         }
 472
 473         /* Configure RSS if applicable */
 474         fm10k_dev_mq_rx_configure(dev);
 475         return 0;
 476 }
 477
 478 static int
 479 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 480 {
 481         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 482         int err = -1;
 483         uint32_t reg;
 484         struct fm10k_rx_queue *rxq;
 485
 486         PMD_INIT_FUNC_TRACE();
 487
 488         if (rx_queue_id < dev->data->nb_rx_queues) {
 489                 rxq = dev->data->rx_queues[rx_queue_id];
 490                 err = rx_queue_reset(rxq);
 491                 if (err == -ENOMEM) {
 492                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 493                         return err;
 494                 } else if (err == -EINVAL) {
 495                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 496                                 " %d", err);
 497                         return err;
 498                 }
 499
 500                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 501                  * Note: this must be done AFTER the queue is enabled on real
 502                  * hardware, but BEFORE the queue is enabled when using the
 503                  * emulation platform. Do it in both places for now and remove
 504                  * this comment and the following two register writes when the
 505                  * emulation platform is no longer being used.
 506                  */
 507                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 508                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 509
 510                 /* Set PF ownership flag for PF devices */
 511                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 512                 if (hw->mac.type == fm10k_mac_pf)
 513                         reg |= FM10K_RXQCTL_PF;
 514                 reg |= FM10K_RXQCTL_ENABLE;
 515                 /* enable RX queue */
 516                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 517                 FM10K_WRITE_FLUSH(hw);
 518
 519                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 520                  * Note: this must be done AFTER the queue is enabled
 521                  */
 522                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 523                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 524         }
 525
 526         return err;
 527 }
 528
 529 static int
 530 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 531 {
 532         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 533
 534         PMD_INIT_FUNC_TRACE();
 535
 536         if (rx_queue_id < dev->data->nb_rx_queues) {
 537                 /* Disable RX queue */
 538                 rx_queue_disable(hw, rx_queue_id);
 539
 540                 /* Free mbuf and clean HW ring */
 541                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 542         }
 543
 544         return 0;
 545 }
 546
 547 static int
 548 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 549 {
 550         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 551         /** @todo - this should be defined in the shared code */
 552 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 553         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 554         int err = 0;
 555
 556         PMD_INIT_FUNC_TRACE();
 557
 558         if (tx_queue_id < dev->data->nb_tx_queues) {
 559                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 560
 561                 /* reset head and tail pointers */
 562                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 563                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 564
 565                 /* enable TX queue */
 566                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 567                                         FM10K_TXDCTL_ENABLE | txdctl);
 568                 FM10K_WRITE_FLUSH(hw);
 569         } else
 570                 err = -1;
 571
 572         return err;
 573 }
 574
 575 static int
 576 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 577 {
 578         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 579
 580         PMD_INIT_FUNC_TRACE();
 581
 582         if (tx_queue_id < dev->data->nb_tx_queues) {
 583                 tx_queue_disable(hw, tx_queue_id);
 584                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 585         }
 586
 587         return 0;
 588 }
 589
 590 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 591 {
 592         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 593                 != FM10K_DGLORTMAP_NONE);
 594 }
 595
 596 static void
 597 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 598 {
 599         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 600         int status;
 601
 602         PMD_INIT_FUNC_TRACE();
 603
 604         /* Return if it didn't acquire valid glort range */
 605         if (!fm10k_glort_valid(hw))
 606                 return;
 607
 608         fm10k_mbx_lock(hw);
 609         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 610                                 FM10K_XCAST_MODE_PROMISC);
 611         fm10k_mbx_unlock(hw);
 612
 613         if (status != FM10K_SUCCESS)
 614                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 615 }
 616
 617 static void
 618 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 619 {
 620         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 621         uint8_t mode;
 622         int status;
 623
 624         PMD_INIT_FUNC_TRACE();
 625
 626         /* Return if it didn't acquire valid glort range */
 627         if (!fm10k_glort_valid(hw))
 628                 return;
 629
 630         if (dev->data->all_multicast == 1)
 631                 mode = FM10K_XCAST_MODE_ALLMULTI;
 632         else
 633                 mode = FM10K_XCAST_MODE_NONE;
 634
 635         fm10k_mbx_lock(hw);
 636         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 637                                 mode);
 638         fm10k_mbx_unlock(hw);
 639
 640         if (status != FM10K_SUCCESS)
 641                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 642 }
 643
 644 static void
 645 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 646 {
 647         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 648         int status;
 649
 650         PMD_INIT_FUNC_TRACE();
 651
 652         /* Return if it didn't acquire valid glort range */
 653         if (!fm10k_glort_valid(hw))
 654                 return;
 655
 656         /* If promiscuous mode is enabled, it doesn't make sense to enable
 657          * allmulticast and disable promiscuous since fm10k only can select
 658          * one of the modes.
 659          */
 660         if (dev->data->promiscuous) {
 661                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 662                         "needn't enable allmulticast");
 663                 return;
 664         }
 665
 666         fm10k_mbx_lock(hw);
 667         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 668                                 FM10K_XCAST_MODE_ALLMULTI);
 669         fm10k_mbx_unlock(hw);
 670
 671         if (status != FM10K_SUCCESS)
 672                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 673 }
 674
 675 static void
 676 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 677 {
 678         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 679         int status;
 680
 681         PMD_INIT_FUNC_TRACE();
 682
 683         /* Return if it didn't acquire valid glort range */
 684         if (!fm10k_glort_valid(hw))
 685                 return;
 686
 687         if (dev->data->promiscuous) {
 688                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 689                         "since promisc mode is enabled");
 690                 return;
 691         }
 692
 693         fm10k_mbx_lock(hw);
 694         /* Change mode to unicast mode */
 695         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 696                                 FM10K_XCAST_MODE_NONE);
 697         fm10k_mbx_unlock(hw);
 698
 699         if (status != FM10K_SUCCESS)
 700                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 701 }
 702
 703 /* fls = find last set bit = 32 minus the number of leading zeros */
 704 #ifndef fls
 705 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 706 #endif
 707 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 708 static int
 709 fm10k_dev_start(struct rte_eth_dev *dev)
 710 {
 711         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 712         int i, diag;
 713
 714         PMD_INIT_FUNC_TRACE();
 715
 716         /* stop, init, then start the hw */
 717         diag = fm10k_stop_hw(hw);
 718         if (diag != FM10K_SUCCESS) {
 719                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 720                 return -EIO;
 721         }
 722
 723         diag = fm10k_init_hw(hw);
 724         if (diag != FM10K_SUCCESS) {
 725                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 726                 return -EIO;
 727         }
 728
 729         diag = fm10k_start_hw(hw);
 730         if (diag != FM10K_SUCCESS) {
 731                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 732                 return -EIO;
 733         }
 734
 735         diag = fm10k_dev_tx_init(dev);
 736         if (diag) {
 737                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 738                 return diag;
 739         }
 740
 741         diag = fm10k_dev_rx_init(dev);
 742         if (diag) {
 743                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 744                 return diag;
 745         }
 746
 747         if (hw->mac.type == fm10k_mac_pf) {
 748                 /* Establish only VSI 0 as valid */
 749                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 750
 751                 /* Configure RSS bits used in RETA table */
 752                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 753                                 fls(dev->data->nb_rx_queues - 1) <<
 754                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 755
 756                 /* Invalidate all other GLORT entries */
 757                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 758                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 759                                         FM10K_DGLORTMAP_NONE);
 760         }
 761
 762         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 763                 struct fm10k_rx_queue *rxq;
 764                 rxq = dev->data->rx_queues[i];
 765
 766                 if (rxq->rx_deferred_start)
 767                         continue;
 768                 diag = fm10k_dev_rx_queue_start(dev, i);
 769                 if (diag != 0) {
 770                         int j;
 771                         for (j = 0; j < i; ++j)
 772                                 rx_queue_clean(dev->data->rx_queues[j]);
 773                         return diag;
 774                 }
 775         }
 776
 777         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 778                 struct fm10k_tx_queue *txq;
 779                 txq = dev->data->tx_queues[i];
 780
 781                 if (txq->tx_deferred_start)
 782                         continue;
 783                 diag = fm10k_dev_tx_queue_start(dev, i);
 784                 if (diag != 0) {
 785                         int j;
 786                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 787                                 rx_queue_clean(dev->data->rx_queues[j]);
 788                         return diag;
 789                 }
 790         }
 791
 792         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID) {
 793                 /* Update default vlan */
 794                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 795
 796                 /* Add default mac/vlan filter to PF/Switch manager */
 797                 fm10k_MAC_filter_set(dev, hw->mac.addr, true);
 798         }
 799
 800         return 0;
 801 }
 802
 803 static void
 804 fm10k_dev_stop(struct rte_eth_dev *dev)
 805 {
 806         int i;
 807
 808         PMD_INIT_FUNC_TRACE();
 809
 810         for (i = 0; i < dev->data->nb_tx_queues; i++)
 811                 fm10k_dev_tx_queue_stop(dev, i);
 812
 813         for (i = 0; i < dev->data->nb_rx_queues; i++)
 814                 fm10k_dev_rx_queue_stop(dev, i);
 815 }
 816
 817 static void
 818 fm10k_dev_close(struct rte_eth_dev *dev)
 819 {
 820         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 821
 822         PMD_INIT_FUNC_TRACE();
 823
 824         /* Stop mailbox service first */
 825         fm10k_close_mbx_service(hw);
 826         fm10k_dev_stop(dev);
 827         fm10k_stop_hw(hw);
 828 }
 829
 830 static int
 831 fm10k_link_update(struct rte_eth_dev *dev,
 832         __rte_unused int wait_to_complete)
 833 {
 834         PMD_INIT_FUNC_TRACE();
 835
 836         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 837          * x8 PCIe interface. For now, we leave the speed undefined since there
 838          * is no 50Gbps Ethernet. */
 839         dev->data->dev_link.link_speed  = 0;
 840         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 841         dev->data->dev_link.link_status = 1;
 842
 843         return 0;
 844 }
 845
 846 static void
 847 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 848 {
 849         uint64_t ipackets, opackets, ibytes, obytes;
 850         struct fm10k_hw *hw =
 851                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 852         struct fm10k_hw_stats *hw_stats =
 853                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 854         int i;
 855
 856         PMD_INIT_FUNC_TRACE();
 857
 858         fm10k_update_hw_stats(hw, hw_stats);
 859
 860         ipackets = opackets = ibytes = obytes = 0;
 861         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 862                 (i < hw->mac.max_queues); ++i) {
 863                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 864                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 865                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 866                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 867                 ipackets += stats->q_ipackets[i];
 868                 opackets += stats->q_opackets[i];
 869                 ibytes   += stats->q_ibytes[i];
 870                 obytes   += stats->q_obytes[i];
 871         }
 872         stats->ipackets = ipackets;
 873         stats->opackets = opackets;
 874         stats->ibytes = ibytes;
 875         stats->obytes = obytes;
 876 }
 877
 878 static void
 879 fm10k_stats_reset(struct rte_eth_dev *dev)
 880 {
 881         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 882         struct fm10k_hw_stats *hw_stats =
 883                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 884
 885         PMD_INIT_FUNC_TRACE();
 886
 887         memset(hw_stats, 0, sizeof(*hw_stats));
 888         fm10k_rebind_hw_stats(hw, hw_stats);
 889 }
 890
 891 static void
 892 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 893         struct rte_eth_dev_info *dev_info)
 894 {
 895         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 896
 897         PMD_INIT_FUNC_TRACE();
 898
 899         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 900         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 901         dev_info->max_rx_queues      = hw->mac.max_queues;
 902         dev_info->max_tx_queues      = hw->mac.max_queues;
 903         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
 904         dev_info->max_hash_mac_addrs = 0;
 905         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 906         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 907         dev_info->rx_offload_capa =
 908                 DEV_RX_OFFLOAD_VLAN_STRIP |
 909                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 910                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 911                 DEV_RX_OFFLOAD_TCP_CKSUM;
 912         dev_info->tx_offload_capa =
 913                 DEV_TX_OFFLOAD_VLAN_INSERT;
 914         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 915
 916         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 917                 .rx_thresh = {
 918                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 919                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 920                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 921                 },
 922                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 923                 .rx_drop_en = 0,
 924         };
 925
 926         dev_info->default_txconf = (struct rte_eth_txconf) {
 927                 .tx_thresh = {
 928                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 929                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 930                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 931                 },
 932                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 933                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 934                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 935                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 936         };
 937
 938 }
 939
 940 static int
 941 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 942 {
 943         s32 result;
 944         uint16_t mac_num = 0;
 945         uint32_t vid_idx, vid_bit, mac_index;
 946         struct fm10k_hw *hw;
 947         struct fm10k_macvlan_filter_info *macvlan;
 948         struct rte_eth_dev_data *data = dev->data;
 949
 950         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 951         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 952
 953         /* @todo - add support for the VF */
 954         if (hw->mac.type != fm10k_mac_pf) {
 955                 PMD_INIT_LOG(ERR, "VLAN filter not available on VF");
 956                 return -ENOTSUP;
 957         }
 958
 959         if (vlan_id > ETH_VLAN_ID_MAX) {
 960                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
 961                 return (-EINVAL);
 962         }
 963
 964         vid_idx = FM10K_VFTA_IDX(vlan_id);
 965         vid_bit = FM10K_VFTA_BIT(vlan_id);
 966         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
 967         if (on && (macvlan->vfta[vid_idx] & vid_bit))
 968                 return 0;
 969         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
 970         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
 971                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
 972                         "in the VLAN filter table");
 973                 return (-EINVAL);
 974         }
 975
 976         fm10k_mbx_lock(hw);
 977         result = fm10k_update_vlan(hw, vlan_id, 0, on);
 978         fm10k_mbx_unlock(hw);
 979         if (result != FM10K_SUCCESS) {
 980                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
 981                 return (-EIO);
 982         }
 983
 984         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
 985                         (result == FM10K_SUCCESS); mac_index++) {
 986                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
 987                         continue;
 988                 if (mac_num > macvlan->mac_num - 1) {
 989                         PMD_INIT_LOG(ERR, "MAC address number "
 990                                         "not match");
 991                         break;
 992                 }
 993                 fm10k_mbx_lock(hw);
 994                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
 995                         data->mac_addrs[mac_index].addr_bytes,
 996                         vlan_id, on, 0);
 997                 fm10k_mbx_unlock(hw);
 998                 mac_num++;
 999         }
1000         if (result != FM10K_SUCCESS) {
1001                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1002                 return (-EIO);
1003         }
1004
1005         if (on) {
1006                 macvlan->vlan_num++;
1007                 macvlan->vfta[vid_idx] |= vid_bit;
1008         } else {
1009                 macvlan->vlan_num--;
1010                 macvlan->vfta[vid_idx] &= ~vid_bit;
1011         }
1012         return 0;
1013 }
1014
1015 static void
1016 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1017 {
1018         if (mask & ETH_VLAN_STRIP_MASK) {
1019                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1020                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1021                                         "always on in fm10k");
1022         }
1023
1024         if (mask & ETH_VLAN_EXTEND_MASK) {
1025                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1026                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1027                                         "supported in fm10k");
1028         }
1029
1030         if (mask & ETH_VLAN_FILTER_MASK) {
1031                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1032                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1033         }
1034 }
1035
1036 /* Add/Remove a MAC address, and update filters */
1037 static void
1038 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add)
1039 {
1040         uint32_t i, j, k;
1041         struct fm10k_hw *hw;
1042         struct fm10k_macvlan_filter_info *macvlan;
1043
1044         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1045         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1046
1047         /* @todo - add support for the VF */
1048         if (hw->mac.type != fm10k_mac_pf) {
1049                 PMD_INIT_LOG(ERR, "MAC filter not available on VF");
1050                 return;
1051         }
1052
1053         i = 0;
1054         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1055                 if (macvlan->vfta[j]) {
1056                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1057                                 if (macvlan->vfta[j] & (1 << k)) {
1058                                         if (i + 1 > macvlan->vlan_num) {
1059                                                 PMD_INIT_LOG(ERR, "vlan number "
1060                                                                 "not match");
1061                                                 return;
1062                                         }
1063                                         fm10k_mbx_lock(hw);
1064                                         fm10k_update_uc_addr(hw,
1065                                                 hw->mac.dglort_map, mac,
1066                                                 j * FM10K_UINT32_BIT_SIZE + k,
1067                                                 add, 0);
1068                                         fm10k_mbx_unlock(hw);
1069                                         i++;
1070                                 }
1071                         }
1072                 }
1073         }
1074
1075         if (add)
1076                 macvlan->mac_num++;
1077         else
1078                 macvlan->mac_num--;
1079 }
1080
1081 /* Add a MAC address, and update filters */
1082 static void
1083 fm10k_macaddr_add(struct rte_eth_dev *dev,
1084                  struct ether_addr *mac_addr,
1085                  __rte_unused uint32_t index,
1086                  __rte_unused uint32_t pool)
1087 {
1088         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE);
1089 }
1090
1091 /* Remove a MAC address, and update filters */
1092 static void
1093 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1094 {
1095         struct rte_eth_dev_data *data = dev->data;
1096
1097         if (index < FM10K_MAX_MACADDR_NUM)
1098                 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1099                                 FALSE);
1100 }
1101
1102 static inline int
1103 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1104 {
1105         if ((request < min) || (request > max) || ((request % mult) != 0))
1106                 return -1;
1107         else
1108                 return 0;
1109 }
1110
1111 /*
1112  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1113  * the physical address is required. If the memzone is already created, then
1114  * this function returns a pointer to the existing memzone.
1115  */
1116 static inline const struct rte_memzone *
1117 allocate_hw_ring(const char *driver_name, const char *ring_name,
1118         uint8_t port_id, uint16_t queue_id, int socket_id,
1119         uint32_t size, uint32_t align)
1120 {
1121         char name[RTE_MEMZONE_NAMESIZE];
1122         const struct rte_memzone *mz;
1123
1124         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1125                  driver_name, ring_name, port_id, queue_id, socket_id);
1126
1127         /* return the memzone if it already exists */
1128         mz = rte_memzone_lookup(name);
1129         if (mz)
1130                 return mz;
1131
1132 #ifdef RTE_LIBRTE_XEN_DOM0
1133         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1134                                            RTE_PGSIZE_2M);
1135 #else
1136         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1137 #endif
1138 }
1139
1140 static inline int
1141 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1142 {
1143         if ((request < min) || (request > max) || ((div % request) != 0))
1144                 return -1;
1145         else
1146                 return 0;
1147 }
1148
1149 static inline int
1150 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1151 {
1152         uint16_t rx_free_thresh;
1153
1154         if (conf->rx_free_thresh == 0)
1155                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1156         else
1157                 rx_free_thresh = conf->rx_free_thresh;
1158
1159         /* make sure the requested threshold satisfies the constraints */
1160         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1161                         FM10K_RX_FREE_THRESH_MAX(q),
1162                         FM10K_RX_FREE_THRESH_DIV(q),
1163                         rx_free_thresh)) {
1164                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1165                         "less than or equal to %u, "
1166                         "greater than or equal to %u, "
1167                         "and a divisor of %u",
1168                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1169                         FM10K_RX_FREE_THRESH_MIN(q),
1170                         FM10K_RX_FREE_THRESH_DIV(q));
1171                 return (-EINVAL);
1172         }
1173
1174         q->alloc_thresh = rx_free_thresh;
1175         q->drop_en = conf->rx_drop_en;
1176         q->rx_deferred_start = conf->rx_deferred_start;
1177
1178         return 0;
1179 }
1180
1181 /*
1182  * Hardware requires specific alignment for Rx packet buffers. At
1183  * least one of the following two conditions must be satisfied.
1184  *  1. Address is 512B aligned
1185  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1186  *
1187  * As such, the driver may need to adjust the DMA address within the
1188  * buffer by up to 512B.
1189  *
1190  * return 1 if the element size is valid, otherwise return 0.
1191  */
1192 static int
1193 mempool_element_size_valid(struct rte_mempool *mp)
1194 {
1195         uint32_t min_size;
1196
1197         /* elt_size includes mbuf header and headroom */
1198         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1199                         RTE_PKTMBUF_HEADROOM;
1200
1201         /* account for up to 512B of alignment */
1202         min_size -= FM10K_RX_DATABUF_ALIGN;
1203
1204         /* sanity check for overflow */
1205         if (min_size > mp->elt_size)
1206                 return 0;
1207
1208         /* size is valid */
1209         return 1;
1210 }
1211
1212 static int
1213 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1214         uint16_t nb_desc, unsigned int socket_id,
1215         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1216 {
1217         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1218         struct fm10k_rx_queue *q;
1219         const struct rte_memzone *mz;
1220
1221         PMD_INIT_FUNC_TRACE();
1222
1223         /* make sure the mempool element size can account for alignment. */
1224         if (!mempool_element_size_valid(mp)) {
1225                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1226                 return (-EINVAL);
1227         }
1228
1229         /* make sure a valid number of descriptors have been requested */
1230         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1231                                 FM10K_MULT_RX_DESC, nb_desc)) {
1232                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1233                         "less than or equal to %"PRIu32", "
1234                         "greater than or equal to %u, "
1235                         "and a multiple of %u",
1236                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1237                         FM10K_MULT_RX_DESC);
1238                 return (-EINVAL);
1239         }
1240
1241         /*
1242          * if this queue existed already, free the associated memory. The
1243          * queue cannot be reused in case we need to allocate memory on
1244          * different socket than was previously used.
1245          */
1246         if (dev->data->rx_queues[queue_id] != NULL) {
1247                 rx_queue_free(dev->data->rx_queues[queue_id]);
1248                 dev->data->rx_queues[queue_id] = NULL;
1249         }
1250
1251         /* allocate memory for the queue structure */
1252         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1253                                 socket_id);
1254         if (q == NULL) {
1255                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1256                 return (-ENOMEM);
1257         }
1258
1259         /* setup queue */
1260         q->mp = mp;
1261         q->nb_desc = nb_desc;
1262         q->port_id = dev->data->port_id;
1263         q->queue_id = queue_id;
1264         q->tail_ptr = (volatile uint32_t *)
1265                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1266         if (handle_rxconf(q, conf))
1267                 return (-EINVAL);
1268
1269         /* allocate memory for the software ring */
1270         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1271                                         nb_desc * sizeof(struct rte_mbuf *),
1272                                         RTE_CACHE_LINE_SIZE, socket_id);
1273         if (q->sw_ring == NULL) {
1274                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1275                 rte_free(q);
1276                 return (-ENOMEM);
1277         }
1278
1279         /*
1280          * allocate memory for the hardware descriptor ring. A memzone large
1281          * enough to hold the maximum ring size is requested to allow for
1282          * resizing in later calls to the queue setup function.
1283          */
1284         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1285                                 dev->data->port_id, queue_id, socket_id,
1286                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1287         if (mz == NULL) {
1288                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1289                 rte_free(q->sw_ring);
1290                 rte_free(q);
1291                 return (-ENOMEM);
1292         }
1293         q->hw_ring = mz->addr;
1294         q->hw_ring_phys_addr = mz->phys_addr;
1295
1296         dev->data->rx_queues[queue_id] = q;
1297         return 0;
1298 }
1299
1300 static void
1301 fm10k_rx_queue_release(void *queue)
1302 {
1303         PMD_INIT_FUNC_TRACE();
1304
1305         rx_queue_free(queue);
1306 }
1307
1308 static inline int
1309 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1310 {
1311         uint16_t tx_free_thresh;
1312         uint16_t tx_rs_thresh;
1313
1314         /* constraint MACROs require that tx_free_thresh is configured
1315          * before tx_rs_thresh */
1316         if (conf->tx_free_thresh == 0)
1317                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1318         else
1319                 tx_free_thresh = conf->tx_free_thresh;
1320
1321         /* make sure the requested threshold satisfies the constraints */
1322         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1323                         FM10K_TX_FREE_THRESH_MAX(q),
1324                         FM10K_TX_FREE_THRESH_DIV(q),
1325                         tx_free_thresh)) {
1326                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1327                         "less than or equal to %u, "
1328                         "greater than or equal to %u, "
1329                         "and a divisor of %u",
1330                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1331                         FM10K_TX_FREE_THRESH_MIN(q),
1332                         FM10K_TX_FREE_THRESH_DIV(q));
1333                 return (-EINVAL);
1334         }
1335
1336         q->free_thresh = tx_free_thresh;
1337
1338         if (conf->tx_rs_thresh == 0)
1339                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1340         else
1341                 tx_rs_thresh = conf->tx_rs_thresh;
1342
1343         q->tx_deferred_start = conf->tx_deferred_start;
1344
1345         /* make sure the requested threshold satisfies the constraints */
1346         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1347                         FM10K_TX_RS_THRESH_MAX(q),
1348                         FM10K_TX_RS_THRESH_DIV(q),
1349                         tx_rs_thresh)) {
1350                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1351                         "less than or equal to %u, "
1352                         "greater than or equal to %u, "
1353                         "and a divisor of %u",
1354                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1355                         FM10K_TX_RS_THRESH_MIN(q),
1356                         FM10K_TX_RS_THRESH_DIV(q));
1357                 return (-EINVAL);
1358         }
1359
1360         q->rs_thresh = tx_rs_thresh;
1361
1362         return 0;
1363 }
1364
1365 static int
1366 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1367         uint16_t nb_desc, unsigned int socket_id,
1368         const struct rte_eth_txconf *conf)
1369 {
1370         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1371         struct fm10k_tx_queue *q;
1372         const struct rte_memzone *mz;
1373
1374         PMD_INIT_FUNC_TRACE();
1375
1376         /* make sure a valid number of descriptors have been requested */
1377         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1378                                 FM10K_MULT_TX_DESC, nb_desc)) {
1379                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1380                         "less than or equal to %"PRIu32", "
1381                         "greater than or equal to %u, "
1382                         "and a multiple of %u",
1383                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1384                         FM10K_MULT_TX_DESC);
1385                 return (-EINVAL);
1386         }
1387
1388         /*
1389          * if this queue existed already, free the associated memory. The
1390          * queue cannot be reused in case we need to allocate memory on
1391          * different socket than was previously used.
1392          */
1393         if (dev->data->tx_queues[queue_id] != NULL) {
1394                 tx_queue_free(dev->data->tx_queues[queue_id]);
1395                 dev->data->tx_queues[queue_id] = NULL;
1396         }
1397
1398         /* allocate memory for the queue structure */
1399         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1400                                 socket_id);
1401         if (q == NULL) {
1402                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1403                 return (-ENOMEM);
1404         }
1405
1406         /* setup queue */
1407         q->nb_desc = nb_desc;
1408         q->port_id = dev->data->port_id;
1409         q->queue_id = queue_id;
1410         q->tail_ptr = (volatile uint32_t *)
1411                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1412         if (handle_txconf(q, conf))
1413                 return (-EINVAL);
1414
1415         /* allocate memory for the software ring */
1416         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1417                                         nb_desc * sizeof(struct rte_mbuf *),
1418                                         RTE_CACHE_LINE_SIZE, socket_id);
1419         if (q->sw_ring == NULL) {
1420                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1421                 rte_free(q);
1422                 return (-ENOMEM);
1423         }
1424
1425         /*
1426          * allocate memory for the hardware descriptor ring. A memzone large
1427          * enough to hold the maximum ring size is requested to allow for
1428          * resizing in later calls to the queue setup function.
1429          */
1430         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1431                                 dev->data->port_id, queue_id, socket_id,
1432                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1433         if (mz == NULL) {
1434                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1435                 rte_free(q->sw_ring);
1436                 rte_free(q);
1437                 return (-ENOMEM);
1438         }
1439         q->hw_ring = mz->addr;
1440         q->hw_ring_phys_addr = mz->phys_addr;
1441
1442         /*
1443          * allocate memory for the RS bit tracker. Enough slots to hold the
1444          * descriptor index for each RS bit needing to be set are required.
1445          */
1446         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1447                                 ((nb_desc + 1) / q->rs_thresh) *
1448                                 sizeof(uint16_t),
1449                                 RTE_CACHE_LINE_SIZE, socket_id);
1450         if (q->rs_tracker.list == NULL) {
1451                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1452                 rte_free(q->sw_ring);
1453                 rte_free(q);
1454                 return (-ENOMEM);
1455         }
1456
1457         dev->data->tx_queues[queue_id] = q;
1458         return 0;
1459 }
1460
1461 static void
1462 fm10k_tx_queue_release(void *queue)
1463 {
1464         PMD_INIT_FUNC_TRACE();
1465
1466         tx_queue_free(queue);
1467 }
1468
1469 static int
1470 fm10k_reta_update(struct rte_eth_dev *dev,
1471                         struct rte_eth_rss_reta_entry64 *reta_conf,
1472                         uint16_t reta_size)
1473 {
1474         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1475         uint16_t i, j, idx, shift;
1476         uint8_t mask;
1477         uint32_t reta;
1478
1479         PMD_INIT_FUNC_TRACE();
1480
1481         if (reta_size > FM10K_MAX_RSS_INDICES) {
1482                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1483                         "(%d) doesn't match the number hardware can supported "
1484                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1485                 return -EINVAL;
1486         }
1487
1488         /*
1489          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1490          * 128-entries in 32 registers
1491          */
1492         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1493                 idx = i / RTE_RETA_GROUP_SIZE;
1494                 shift = i % RTE_RETA_GROUP_SIZE;
1495                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1496                                 BIT_MASK_PER_UINT32);
1497                 if (mask == 0)
1498                         continue;
1499
1500                 reta = 0;
1501                 if (mask != BIT_MASK_PER_UINT32)
1502                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1503
1504                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1505                         if (mask & (0x1 << j)) {
1506                                 if (mask != 0xF)
1507                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1508                                 reta |= reta_conf[idx].reta[shift + j] <<
1509                                                 (CHAR_BIT * j);
1510                         }
1511                 }
1512                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1513         }
1514
1515         return 0;
1516 }
1517
1518 static int
1519 fm10k_reta_query(struct rte_eth_dev *dev,
1520                         struct rte_eth_rss_reta_entry64 *reta_conf,
1521                         uint16_t reta_size)
1522 {
1523         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1524         uint16_t i, j, idx, shift;
1525         uint8_t mask;
1526         uint32_t reta;
1527
1528         PMD_INIT_FUNC_TRACE();
1529
1530         if (reta_size < FM10K_MAX_RSS_INDICES) {
1531                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1532                         "(%d) doesn't match the number hardware can supported "
1533                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1534                 return -EINVAL;
1535         }
1536
1537         /*
1538          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1539          * 128-entries in 32 registers
1540          */
1541         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1542                 idx = i / RTE_RETA_GROUP_SIZE;
1543                 shift = i % RTE_RETA_GROUP_SIZE;
1544                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1545                                 BIT_MASK_PER_UINT32);
1546                 if (mask == 0)
1547                         continue;
1548
1549                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1550                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1551                         if (mask & (0x1 << j))
1552                                 reta_conf[idx].reta[shift + j] = ((reta >>
1553                                         CHAR_BIT * j) & UINT8_MAX);
1554                 }
1555         }
1556
1557         return 0;
1558 }
1559
1560 static int
1561 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1562         struct rte_eth_rss_conf *rss_conf)
1563 {
1564         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1565         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1566         uint32_t mrqc;
1567         uint64_t hf = rss_conf->rss_hf;
1568         int i;
1569
1570         PMD_INIT_FUNC_TRACE();
1571
1572         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1573                 FM10K_RSSRK_ENTRIES_PER_REG)
1574                 return -EINVAL;
1575
1576         if (hf == 0)
1577                 return -EINVAL;
1578
1579         mrqc = 0;
1580         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1581         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1582         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1583         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1584         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1585         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1586         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1587         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1588         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1589
1590         /* If the mapping doesn't fit any supported, return */
1591         if (mrqc == 0)
1592                 return -EINVAL;
1593
1594         if (key != NULL)
1595                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1596                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1597
1598         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1599
1600         return 0;
1601 }
1602
1603 static int
1604 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1605         struct rte_eth_rss_conf *rss_conf)
1606 {
1607         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1608         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1609         uint32_t mrqc;
1610         uint64_t hf;
1611         int i;
1612
1613         PMD_INIT_FUNC_TRACE();
1614
1615         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1616                                 FM10K_RSSRK_ENTRIES_PER_REG)
1617                 return -EINVAL;
1618
1619         if (key != NULL)
1620                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1621                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1622
1623         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1624         hf = 0;
1625         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1626         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1627         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1628         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1629         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1630         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1631         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1632         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1633         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1634
1635         rss_conf->rss_hf = hf;
1636
1637         return 0;
1638 }
1639
1640 static void
1641 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1642 {
1643         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1644         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1645
1646         /* Bind all local non-queue interrupt to vector 0 */
1647         int_map |= 0;
1648
1649         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1650         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1651         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1652         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1653         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1654         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1655
1656         /* Enable misc causes */
1657         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1658                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1659                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1660                                 FM10K_EIMR_ENABLE(MAILBOX) |
1661                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1662                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1663                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1664                                 FM10K_EIMR_ENABLE(VFLR));
1665
1666         /* Enable ITR 0 */
1667         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1668                                         FM10K_ITR_MASK_CLEAR);
1669         FM10K_WRITE_FLUSH(hw);
1670 }
1671
1672 static void
1673 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1674 {
1675         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1676         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1677
1678         /* Bind all local non-queue interrupt to vector 0 */
1679         int_map |= 0;
1680
1681         /* Only INT 0 available, other 15 are reserved. */
1682         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1683
1684         /* Enable ITR 0 */
1685         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1686                                         FM10K_ITR_MASK_CLEAR);
1687         FM10K_WRITE_FLUSH(hw);
1688 }
1689
1690 static int
1691 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1692 {
1693         struct fm10k_fault fault;
1694         int err;
1695         const char *estr = "Unknown error";
1696
1697         /* Process PCA fault */
1698         if (eicr & FM10K_EIMR_PCA_FAULT) {
1699                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1700                 if (err)
1701                         goto error;
1702                 switch (fault.type) {
1703                 case PCA_NO_FAULT:
1704                         estr = "PCA_NO_FAULT"; break;
1705                 case PCA_UNMAPPED_ADDR:
1706                         estr = "PCA_UNMAPPED_ADDR"; break;
1707                 case PCA_BAD_QACCESS_PF:
1708                         estr = "PCA_BAD_QACCESS_PF"; break;
1709                 case PCA_BAD_QACCESS_VF:
1710                         estr = "PCA_BAD_QACCESS_VF"; break;
1711                 case PCA_MALICIOUS_REQ:
1712                         estr = "PCA_MALICIOUS_REQ"; break;
1713                 case PCA_POISONED_TLP:
1714                         estr = "PCA_POISONED_TLP"; break;
1715                 case PCA_TLP_ABORT:
1716                         estr = "PCA_TLP_ABORT"; break;
1717                 default:
1718                         goto error;
1719                 }
1720                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1721                         estr, fault.func ? "VF" : "PF", fault.func,
1722                         fault.address, fault.specinfo);
1723         }
1724
1725         /* Process THI fault */
1726         if (eicr & FM10K_EIMR_THI_FAULT) {
1727                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1728                 if (err)
1729                         goto error;
1730                 switch (fault.type) {
1731                 case THI_NO_FAULT:
1732                         estr = "THI_NO_FAULT"; break;
1733                 case THI_MAL_DIS_Q_FAULT:
1734                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1735                 default:
1736                         goto error;
1737                 }
1738                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1739                         estr, fault.func ? "VF" : "PF", fault.func,
1740                         fault.address, fault.specinfo);
1741         }
1742
1743         /* Process FUM fault */
1744         if (eicr & FM10K_EIMR_FUM_FAULT) {
1745                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1746                 if (err)
1747                         goto error;
1748                 switch (fault.type) {
1749                 case FUM_NO_FAULT:
1750                         estr = "FUM_NO_FAULT"; break;
1751                 case FUM_UNMAPPED_ADDR:
1752                         estr = "FUM_UNMAPPED_ADDR"; break;
1753                 case FUM_POISONED_TLP:
1754                         estr = "FUM_POISONED_TLP"; break;
1755                 case FUM_BAD_VF_QACCESS:
1756                         estr = "FUM_BAD_VF_QACCESS"; break;
1757                 case FUM_ADD_DECODE_ERR:
1758                         estr = "FUM_ADD_DECODE_ERR"; break;
1759                 case FUM_RO_ERROR:
1760                         estr = "FUM_RO_ERROR"; break;
1761                 case FUM_QPRC_CRC_ERROR:
1762                         estr = "FUM_QPRC_CRC_ERROR"; break;
1763                 case FUM_CSR_TIMEOUT:
1764                         estr = "FUM_CSR_TIMEOUT"; break;
1765                 case FUM_INVALID_TYPE:
1766                         estr = "FUM_INVALID_TYPE"; break;
1767                 case FUM_INVALID_LENGTH:
1768                         estr = "FUM_INVALID_LENGTH"; break;
1769                 case FUM_INVALID_BE:
1770                         estr = "FUM_INVALID_BE"; break;
1771                 case FUM_INVALID_ALIGN:
1772                         estr = "FUM_INVALID_ALIGN"; break;
1773                 default:
1774                         goto error;
1775                 }
1776                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1777                         estr, fault.func ? "VF" : "PF", fault.func,
1778                         fault.address, fault.specinfo);
1779         }
1780
1781         if (estr)
1782                 return 0;
1783         return 0;
1784 error:
1785         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1786         return err;
1787 }
1788
1789 /**
1790  * PF interrupt handler triggered by NIC for handling specific interrupt.
1791  *
1792  * @param handle
1793  *  Pointer to interrupt handle.
1794  * @param param
1795  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1796  *
1797  * @return
1798  *  void
1799  */
1800 static void
1801 fm10k_dev_interrupt_handler_pf(
1802                         __rte_unused struct rte_intr_handle *handle,
1803                         void *param)
1804 {
1805         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1806         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1807         uint32_t cause, status;
1808
1809         if (hw->mac.type != fm10k_mac_pf)
1810                 return;
1811
1812         cause = FM10K_READ_REG(hw, FM10K_EICR);
1813
1814         /* Handle PCI fault cases */
1815         if (cause & FM10K_EICR_FAULT_MASK) {
1816                 PMD_INIT_LOG(ERR, "INT: find fault!");
1817                 fm10k_dev_handle_fault(hw, cause);
1818         }
1819
1820         /* Handle switch up/down */
1821         if (cause & FM10K_EICR_SWITCHNOTREADY)
1822                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1823
1824         if (cause & FM10K_EICR_SWITCHREADY)
1825                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1826
1827         /* Handle mailbox message */
1828         fm10k_mbx_lock(hw);
1829         hw->mbx.ops.process(hw, &hw->mbx);
1830         fm10k_mbx_unlock(hw);
1831
1832         /* Handle SRAM error */
1833         if (cause & FM10K_EICR_SRAMERROR) {
1834                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1835
1836                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1837                 /* Write to clear pending bits */
1838                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1839
1840                 /* Todo: print out error message after shared code  updates */
1841         }
1842
1843         /* Clear these 3 events if having any */
1844         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1845                  FM10K_EICR_SWITCHREADY;
1846         if (cause)
1847                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1848
1849         /* Re-enable interrupt from device side */
1850         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1851                                         FM10K_ITR_MASK_CLEAR);
1852         /* Re-enable interrupt from host side */
1853         rte_intr_enable(&(dev->pci_dev->intr_handle));
1854 }
1855
1856 /**
1857  * VF interrupt handler triggered by NIC for handling specific interrupt.
1858  *
1859  * @param handle
1860  *  Pointer to interrupt handle.
1861  * @param param
1862  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1863  *
1864  * @return
1865  *  void
1866  */
1867 static void
1868 fm10k_dev_interrupt_handler_vf(
1869                         __rte_unused struct rte_intr_handle *handle,
1870                         void *param)
1871 {
1872         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1873         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1874
1875         if (hw->mac.type != fm10k_mac_vf)
1876                 return;
1877
1878         /* Handle mailbox message if lock is acquired */
1879         fm10k_mbx_lock(hw);
1880         hw->mbx.ops.process(hw, &hw->mbx);
1881         fm10k_mbx_unlock(hw);
1882
1883         /* Re-enable interrupt from device side */
1884         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1885                                         FM10K_ITR_MASK_CLEAR);
1886         /* Re-enable interrupt from host side */
1887         rte_intr_enable(&(dev->pci_dev->intr_handle));
1888 }
1889
1890 /* Mailbox message handler in VF */
1891 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1892         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1893         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1894         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1895         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1896 };
1897
1898 /* Mailbox message handler in PF */
1899 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1900         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1901         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1902         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1903         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1904         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1905         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1906         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1907 };
1908
1909 static int
1910 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1911 {
1912         int err;
1913
1914         /* Initialize mailbox lock */
1915         fm10k_mbx_initlock(hw);
1916
1917         /* Replace default message handler with new ones */
1918         if (hw->mac.type == fm10k_mac_pf)
1919                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1920         else
1921                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1922
1923         if (err) {
1924                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1925                                 err);
1926                 return err;
1927         }
1928         /* Connect to SM for PF device or PF for VF device */
1929         return hw->mbx.ops.connect(hw, &hw->mbx);
1930 }
1931
1932 static void
1933 fm10k_close_mbx_service(struct fm10k_hw *hw)
1934 {
1935         /* Disconnect from SM for PF device or PF for VF device */
1936         hw->mbx.ops.disconnect(hw, &hw->mbx);
1937 }
1938
1939 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1940         .dev_configure          = fm10k_dev_configure,
1941         .dev_start              = fm10k_dev_start,
1942         .dev_stop               = fm10k_dev_stop,
1943         .dev_close              = fm10k_dev_close,
1944         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
1945         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
1946         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
1947         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
1948         .stats_get              = fm10k_stats_get,
1949         .stats_reset            = fm10k_stats_reset,
1950         .link_update            = fm10k_link_update,
1951         .dev_infos_get          = fm10k_dev_infos_get,
1952         .vlan_filter_set        = fm10k_vlan_filter_set,
1953         .vlan_offload_set       = fm10k_vlan_offload_set,
1954         .mac_addr_add           = fm10k_macaddr_add,
1955         .mac_addr_remove        = fm10k_macaddr_remove,
1956         .rx_queue_start         = fm10k_dev_rx_queue_start,
1957         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1958         .tx_queue_start         = fm10k_dev_tx_queue_start,
1959         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1960         .rx_queue_setup         = fm10k_rx_queue_setup,
1961         .rx_queue_release       = fm10k_rx_queue_release,
1962         .tx_queue_setup         = fm10k_tx_queue_setup,
1963         .tx_queue_release       = fm10k_tx_queue_release,
1964         .reta_update            = fm10k_reta_update,
1965         .reta_query             = fm10k_reta_query,
1966         .rss_hash_update        = fm10k_rss_hash_update,
1967         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1968 };
1969
1970 static int
1971 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1972 {
1973         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1974         int diag;
1975         struct fm10k_macvlan_filter_info *macvlan;
1976
1977         PMD_INIT_FUNC_TRACE();
1978
1979         dev->dev_ops = &fm10k_eth_dev_ops;
1980         dev->rx_pkt_burst = &fm10k_recv_pkts;
1981         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1982
1983         if (dev->data->scattered_rx)
1984                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1985
1986         /* only initialize in the primary process */
1987         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1988                 return 0;
1989
1990         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1991         memset(macvlan, 0, sizeof(*macvlan));
1992         /* Vendor and Device ID need to be set before init of shared code */
1993         memset(hw, 0, sizeof(*hw));
1994         hw->device_id = dev->pci_dev->id.device_id;
1995         hw->vendor_id = dev->pci_dev->id.vendor_id;
1996         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1997         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1998         hw->revision_id = 0;
1999         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2000         if (hw->hw_addr == NULL) {
2001                 PMD_INIT_LOG(ERR, "Bad mem resource."
2002                         " Try to blacklist unused devices.");
2003                 return -EIO;
2004         }
2005
2006         /* Store fm10k_adapter pointer */
2007         hw->back = dev->data->dev_private;
2008
2009         /* Initialize the shared code */
2010         diag = fm10k_init_shared_code(hw);
2011         if (diag != FM10K_SUCCESS) {
2012                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2013                 return -EIO;
2014         }
2015
2016         /*
2017          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2018          * there is no way to get link status without reading BAR4.  Until this
2019          * works, assume we have maximum bandwidth.
2020          * @todo - fix bus info
2021          */
2022         hw->bus_caps.speed = fm10k_bus_speed_8000;
2023         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2024         hw->bus_caps.payload = fm10k_bus_payload_512;
2025         hw->bus.speed = fm10k_bus_speed_8000;
2026         hw->bus.width = fm10k_bus_width_pcie_x8;
2027         hw->bus.payload = fm10k_bus_payload_256;
2028
2029         /* Initialize the hw */
2030         diag = fm10k_init_hw(hw);
2031         if (diag != FM10K_SUCCESS) {
2032                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2033                 return -EIO;
2034         }
2035
2036         /* Initialize MAC address(es) */
2037         dev->data->mac_addrs = rte_zmalloc("fm10k",
2038                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2039         if (dev->data->mac_addrs == NULL) {
2040                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2041                 return -ENOMEM;
2042         }
2043
2044         diag = fm10k_read_mac_addr(hw);
2045
2046         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2047                         &dev->data->mac_addrs[0]);
2048
2049         if (diag != FM10K_SUCCESS ||
2050                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2051
2052                 /* Generate a random addr */
2053                 eth_random_addr(hw->mac.addr);
2054                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2055                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2056                 &dev->data->mac_addrs[0]);
2057         }
2058
2059         /* Reset the hw statistics */
2060         fm10k_stats_reset(dev);
2061
2062         /* Reset the hw */
2063         diag = fm10k_reset_hw(hw);
2064         if (diag != FM10K_SUCCESS) {
2065                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2066                 return -EIO;
2067         }
2068
2069         /* Setup mailbox service */
2070         diag = fm10k_setup_mbx_service(hw);
2071         if (diag != FM10K_SUCCESS) {
2072                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2073                 return -EIO;
2074         }
2075
2076         /*PF/VF has different interrupt handling mechanism */
2077         if (hw->mac.type == fm10k_mac_pf) {
2078                 /* register callback func to eal lib */
2079                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2080                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2081
2082                 /* enable MISC interrupt */
2083                 fm10k_dev_enable_intr_pf(dev);
2084         } else { /* VF */
2085                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2086                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2087
2088                 fm10k_dev_enable_intr_vf(dev);
2089         }
2090
2091         /* Enable uio intr after callback registered */
2092         rte_intr_enable(&(dev->pci_dev->intr_handle));
2093
2094         hw->mac.ops.update_int_moderator(hw);
2095
2096         /* Make sure Switch Manager is ready before going forward. */
2097         if (hw->mac.type == fm10k_mac_pf) {
2098                 int switch_ready = 0;
2099                 int i;
2100
2101                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2102                         fm10k_mbx_lock(hw);
2103                         hw->mac.ops.get_host_state(hw, &switch_ready);
2104                         fm10k_mbx_unlock(hw);
2105                         if (switch_ready)
2106                                 break;
2107                         /* Delay some time to acquire async LPORT_MAP info. */
2108                         rte_delay_us(WAIT_SWITCH_MSG_US);
2109                 }
2110
2111                 if (switch_ready == 0) {
2112                         PMD_INIT_LOG(ERR, "switch is not ready");
2113                         return -1;
2114                 }
2115         }
2116
2117         /*
2118          * Below function will trigger operations on mailbox, acquire lock to
2119          * avoid race condition from interrupt handler. Operations on mailbox
2120          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2121          * will handle and generate an interrupt to our side. Then,  FIFO in
2122          * mailbox will be touched.
2123          */
2124         fm10k_mbx_lock(hw);
2125         /* Enable port first */
2126         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2127
2128         /*
2129          * Add default mac. glort is assigned by SM for PF, while is
2130          * unused for VF. PF will assign correct glort for VF.
2131          */
2132         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
2133                                 0, 1, 0);
2134
2135         /* Set unicast mode by default. App can change to other mode in other
2136          * API func.
2137          */
2138         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2139                                         FM10K_XCAST_MODE_NONE);
2140
2141         fm10k_mbx_unlock(hw);
2142
2143
2144         return 0;
2145 }
2146
2147 /*
2148  * The set of PCI devices this driver supports. This driver will enable both PF
2149  * and SRIOV-VF devices.
2150  */
2151 static const struct rte_pci_id pci_id_fm10k_map[] = {
2152 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2153 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2154 #include "rte_pci_dev_ids.h"
2155         { .vendor_id = 0, /* sentinel */ },
2156 };
2157
2158 static struct eth_driver rte_pmd_fm10k = {
2159         .pci_drv = {
2160                 .name = "rte_pmd_fm10k",
2161                 .id_table = pci_id_fm10k_map,
2162                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
2163         },
2164         .eth_dev_init = eth_fm10k_dev_init,
2165         .dev_private_size = sizeof(struct fm10k_adapter),
2166 };
2167
2168 /*
2169  * Driver initialization routine.
2170  * Invoked once at EAL init time.
2171  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2172  */
2173 static int
2174 rte_pmd_fm10k_init(__rte_unused const char *name,
2175         __rte_unused const char *params)
2176 {
2177         PMD_INIT_FUNC_TRACE();
2178         rte_eth_driver_register(&rte_pmd_fm10k);
2179         return 0;
2180 }
2181
2182 static struct rte_driver rte_fm10k_driver = {
2183         .type = PMD_PDEV,
2184         .init = rte_pmd_fm10k_init,
2185 };
2186
2187 PMD_REGISTER_DRIVER(rte_fm10k_driver);