drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void
  65 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add);
  66 static void
  67 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev);
  68
  69 static void
  70 fm10k_mbx_initlock(struct fm10k_hw *hw)
  71 {
  72         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  73 }
  74
  75 static void
  76 fm10k_mbx_lock(struct fm10k_hw *hw)
  77 {
  78         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  79                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  80 }
  81
  82 static void
  83 fm10k_mbx_unlock(struct fm10k_hw *hw)
  84 {
  85         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  86 }
  87
  88 /*
  89  * reset queue to initial state, allocate software buffers used when starting
  90  * device.
  91  * return 0 on success
  92  * return -ENOMEM if buffers cannot be allocated
  93  * return -EINVAL if buffers do not satisfy alignment condition
  94  */
  95 static inline int
  96 rx_queue_reset(struct fm10k_rx_queue *q)
  97 {
  98         uint64_t dma_addr;
  99         int i, diag;
 100         PMD_INIT_FUNC_TRACE();
 101
 102         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 103         if (diag != 0)
 104                 return -ENOMEM;
 105
 106         for (i = 0; i < q->nb_desc; ++i) {
 107                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 108                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 109                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 110                                                 q->nb_desc);
 111                         return -EINVAL;
 112                 }
 113                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 114                 q->hw_ring[i].q.pkt_addr = dma_addr;
 115                 q->hw_ring[i].q.hdr_addr = dma_addr;
 116         }
 117
 118         q->next_dd = 0;
 119         q->next_alloc = 0;
 120         q->next_trigger = q->alloc_thresh - 1;
 121         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 122         return 0;
 123 }
 124
 125 /*
 126  * clean queue, descriptor rings, free software buffers used when stopping
 127  * device.
 128  */
 129 static inline void
 130 rx_queue_clean(struct fm10k_rx_queue *q)
 131 {
 132         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 133         uint32_t i;
 134         PMD_INIT_FUNC_TRACE();
 135
 136         /* zero descriptor rings */
 137         for (i = 0; i < q->nb_desc; ++i)
 138                 q->hw_ring[i] = zero;
 139
 140         /* free software buffers */
 141         for (i = 0; i < q->nb_desc; ++i) {
 142                 if (q->sw_ring[i]) {
 143                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 144                         q->sw_ring[i] = NULL;
 145                 }
 146         }
 147 }
 148
 149 /*
 150  * free all queue memory used when releasing the queue (i.e. configure)
 151  */
 152 static inline void
 153 rx_queue_free(struct fm10k_rx_queue *q)
 154 {
 155         PMD_INIT_FUNC_TRACE();
 156         if (q) {
 157                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 158                 rx_queue_clean(q);
 159                 if (q->sw_ring) {
 160                         rte_free(q->sw_ring);
 161                         q->sw_ring = NULL;
 162                 }
 163                 rte_free(q);
 164                 q = NULL;
 165         }
 166 }
 167
 168 /*
 169  * disable RX queue, wait unitl HW finished necessary flush operation
 170  */
 171 static inline int
 172 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 173 {
 174         uint32_t reg, i;
 175
 176         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 177         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 178                         reg & ~FM10K_RXQCTL_ENABLE);
 179
 180         /* Wait 100us at most */
 181         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 182                 rte_delay_us(1);
 183                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 184                 if (!(reg & FM10K_RXQCTL_ENABLE))
 185                         break;
 186         }
 187
 188         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 189                 return -1;
 190
 191         return 0;
 192 }
 193
 194 /*
 195  * reset queue to initial state, allocate software buffers used when starting
 196  * device
 197  */
 198 static inline void
 199 tx_queue_reset(struct fm10k_tx_queue *q)
 200 {
 201         PMD_INIT_FUNC_TRACE();
 202         q->last_free = 0;
 203         q->next_free = 0;
 204         q->nb_used = 0;
 205         q->nb_free = q->nb_desc - 1;
 206         q->free_trigger = q->nb_free - q->free_thresh;
 207         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 208         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 209 }
 210
 211 /*
 212  * clean queue, descriptor rings, free software buffers used when stopping
 213  * device
 214  */
 215 static inline void
 216 tx_queue_clean(struct fm10k_tx_queue *q)
 217 {
 218         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 219         uint32_t i;
 220         PMD_INIT_FUNC_TRACE();
 221
 222         /* zero descriptor rings */
 223         for (i = 0; i < q->nb_desc; ++i)
 224                 q->hw_ring[i] = zero;
 225
 226         /* free software buffers */
 227         for (i = 0; i < q->nb_desc; ++i) {
 228                 if (q->sw_ring[i]) {
 229                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 230                         q->sw_ring[i] = NULL;
 231                 }
 232         }
 233 }
 234
 235 /*
 236  * free all queue memory used when releasing the queue (i.e. configure)
 237  */
 238 static inline void
 239 tx_queue_free(struct fm10k_tx_queue *q)
 240 {
 241         PMD_INIT_FUNC_TRACE();
 242         if (q) {
 243                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 244                 tx_queue_clean(q);
 245                 if (q->rs_tracker.list) {
 246                         rte_free(q->rs_tracker.list);
 247                         q->rs_tracker.list = NULL;
 248                 }
 249                 if (q->sw_ring) {
 250                         rte_free(q->sw_ring);
 251                         q->sw_ring = NULL;
 252                 }
 253                 rte_free(q);
 254                 q = NULL;
 255         }
 256 }
 257
 258 /*
 259  * disable TX queue, wait unitl HW finished necessary flush operation
 260  */
 261 static inline int
 262 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 263 {
 264         uint32_t reg, i;
 265
 266         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 267         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 268                         reg & ~FM10K_TXDCTL_ENABLE);
 269
 270         /* Wait 100us at most */
 271         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 272                 rte_delay_us(1);
 273                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 274                 if (!(reg & FM10K_TXDCTL_ENABLE))
 275                         break;
 276         }
 277
 278         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 279                 return -1;
 280
 281         return 0;
 282 }
 283
 284 static int
 285 fm10k_dev_configure(struct rte_eth_dev *dev)
 286 {
 287         PMD_INIT_FUNC_TRACE();
 288
 289         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 290                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 291
 292         return 0;
 293 }
 294
 295 static void
 296 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 297 {
 298         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 299         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 300         uint32_t mrqc, *key, i, reta, j;
 301         uint64_t hf;
 302
 303 #define RSS_KEY_SIZE 40
 304         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 305                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 306                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 307                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 308                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 309                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 310         };
 311
 312         if (dev->data->nb_rx_queues == 1 ||
 313             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 314             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 315                 return;
 316
 317         /* random key is rss_intel_key (default) or user provided (rss_key) */
 318         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 319                 key = (uint32_t *)rss_intel_key;
 320         else
 321                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 322
 323         /* Now fill our hash function seeds, 4 bytes at a time */
 324         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 325                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 326
 327         /*
 328          * Fill in redirection table
 329          * The byte-swap is needed because NIC registers are in
 330          * little-endian order.
 331          */
 332         reta = 0;
 333         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 334                 if (j == dev->data->nb_rx_queues)
 335                         j = 0;
 336                 reta = (reta << CHAR_BIT) | j;
 337                 if ((i & 3) == 3)
 338                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 339                                         rte_bswap32(reta));
 340         }
 341
 342         /*
 343          * Generate RSS hash based on packet types, TCP/UDP
 344          * port numbers and/or IPv4/v6 src and dst addresses
 345          */
 346         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 347         mrqc = 0;
 348         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 349         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 350         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 351         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 352         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 353         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 354         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 355         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 356         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 357
 358         if (mrqc == 0) {
 359                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 360                         "supported", hf);
 361                 return;
 362         }
 363
 364         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 365 }
 366
 367 static int
 368 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 369 {
 370         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 371         int i, ret;
 372         struct fm10k_tx_queue *txq;
 373         uint64_t base_addr;
 374         uint32_t size;
 375
 376         /* Disable TXINT to avoid possible interrupt */
 377         for (i = 0; i < hw->mac.max_queues; i++)
 378                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 379                                 3 << FM10K_TXINT_TIMER_SHIFT);
 380
 381         /* Setup TX queue */
 382         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 383                 txq = dev->data->tx_queues[i];
 384                 base_addr = txq->hw_ring_phys_addr;
 385                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 386
 387                 /* disable queue to avoid issues while updating state */
 388                 ret = tx_queue_disable(hw, i);
 389                 if (ret) {
 390                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 391                         return -1;
 392                 }
 393
 394                 /* set location and size for descriptor ring */
 395                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 396                                 base_addr & UINT64_LOWER_32BITS_MASK);
 397                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 398                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 399                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 400         }
 401         return 0;
 402 }
 403
 404 static int
 405 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 406 {
 407         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 408         int i, ret;
 409         struct fm10k_rx_queue *rxq;
 410         uint64_t base_addr;
 411         uint32_t size;
 412         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 413         uint16_t buf_size;
 414
 415         /* Disable RXINT to avoid possible interrupt */
 416         for (i = 0; i < hw->mac.max_queues; i++)
 417                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 418                                 3 << FM10K_RXINT_TIMER_SHIFT);
 419
 420         /* Setup RX queues */
 421         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 422                 rxq = dev->data->rx_queues[i];
 423                 base_addr = rxq->hw_ring_phys_addr;
 424                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 425
 426                 /* disable queue to avoid issues while updating state */
 427                 ret = rx_queue_disable(hw, i);
 428                 if (ret) {
 429                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 430                         return -1;
 431                 }
 432
 433                 /* Setup the Base and Length of the Rx Descriptor Ring */
 434                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 435                                 base_addr & UINT64_LOWER_32BITS_MASK);
 436                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 437                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 438                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 439
 440                 /* Configure the Rx buffer size for one buff without split */
 441                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 442                         RTE_PKTMBUF_HEADROOM);
 443                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 444                  * reserved for this purpose, and the worst case could be 511B.
 445                  * But SRR reg assumes all buffers have the same size. In order
 446                  * to fill the gap, we'll have to consider the worst case and
 447                  * assume 512B is reserved. If we don't do so, it's possible
 448                  * for HW to overwrite data to next mbuf.
 449                  */
 450                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 451
 452                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 453                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 454
 455                 /* It adds dual VLAN length for supporting dual VLAN */
 456                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 457                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 458                         dev->data->dev_conf.rxmode.enable_scatter) {
 459                         uint32_t reg;
 460                         dev->data->scattered_rx = 1;
 461                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 462                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 463                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 464                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 465                 }
 466
 467                 /* Enable drop on empty, it's RO for VF */
 468                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 469                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 470
 471                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 472                 FM10K_WRITE_FLUSH(hw);
 473         }
 474
 475         /* Configure RSS if applicable */
 476         fm10k_dev_mq_rx_configure(dev);
 477         return 0;
 478 }
 479
 480 static int
 481 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 482 {
 483         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 484         int err = -1;
 485         uint32_t reg;
 486         struct fm10k_rx_queue *rxq;
 487
 488         PMD_INIT_FUNC_TRACE();
 489
 490         if (rx_queue_id < dev->data->nb_rx_queues) {
 491                 rxq = dev->data->rx_queues[rx_queue_id];
 492                 err = rx_queue_reset(rxq);
 493                 if (err == -ENOMEM) {
 494                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 495                         return err;
 496                 } else if (err == -EINVAL) {
 497                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 498                                 " %d", err);
 499                         return err;
 500                 }
 501
 502                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 503                  * Note: this must be done AFTER the queue is enabled on real
 504                  * hardware, but BEFORE the queue is enabled when using the
 505                  * emulation platform. Do it in both places for now and remove
 506                  * this comment and the following two register writes when the
 507                  * emulation platform is no longer being used.
 508                  */
 509                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 510                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 511
 512                 /* Set PF ownership flag for PF devices */
 513                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 514                 if (hw->mac.type == fm10k_mac_pf)
 515                         reg |= FM10K_RXQCTL_PF;
 516                 reg |= FM10K_RXQCTL_ENABLE;
 517                 /* enable RX queue */
 518                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 519                 FM10K_WRITE_FLUSH(hw);
 520
 521                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 522                  * Note: this must be done AFTER the queue is enabled
 523                  */
 524                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 525                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 526         }
 527
 528         return err;
 529 }
 530
 531 static int
 532 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 533 {
 534         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 535
 536         PMD_INIT_FUNC_TRACE();
 537
 538         if (rx_queue_id < dev->data->nb_rx_queues) {
 539                 /* Disable RX queue */
 540                 rx_queue_disable(hw, rx_queue_id);
 541
 542                 /* Free mbuf and clean HW ring */
 543                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 544         }
 545
 546         return 0;
 547 }
 548
 549 static int
 550 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 551 {
 552         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 553         /** @todo - this should be defined in the shared code */
 554 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 555         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 556         int err = 0;
 557
 558         PMD_INIT_FUNC_TRACE();
 559
 560         if (tx_queue_id < dev->data->nb_tx_queues) {
 561                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 562
 563                 /* reset head and tail pointers */
 564                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 565                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 566
 567                 /* enable TX queue */
 568                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 569                                         FM10K_TXDCTL_ENABLE | txdctl);
 570                 FM10K_WRITE_FLUSH(hw);
 571         } else
 572                 err = -1;
 573
 574         return err;
 575 }
 576
 577 static int
 578 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 579 {
 580         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 581
 582         PMD_INIT_FUNC_TRACE();
 583
 584         if (tx_queue_id < dev->data->nb_tx_queues) {
 585                 tx_queue_disable(hw, tx_queue_id);
 586                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 587         }
 588
 589         return 0;
 590 }
 591
 592 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 593 {
 594         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 595                 != FM10K_DGLORTMAP_NONE);
 596 }
 597
 598 static void
 599 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 600 {
 601         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 602         int status;
 603
 604         PMD_INIT_FUNC_TRACE();
 605
 606         /* Return if it didn't acquire valid glort range */
 607         if (!fm10k_glort_valid(hw))
 608                 return;
 609
 610         fm10k_mbx_lock(hw);
 611         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 612                                 FM10K_XCAST_MODE_PROMISC);
 613         fm10k_mbx_unlock(hw);
 614
 615         if (status != FM10K_SUCCESS)
 616                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 617 }
 618
 619 static void
 620 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 621 {
 622         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 623         uint8_t mode;
 624         int status;
 625
 626         PMD_INIT_FUNC_TRACE();
 627
 628         /* Return if it didn't acquire valid glort range */
 629         if (!fm10k_glort_valid(hw))
 630                 return;
 631
 632         if (dev->data->all_multicast == 1)
 633                 mode = FM10K_XCAST_MODE_ALLMULTI;
 634         else
 635                 mode = FM10K_XCAST_MODE_NONE;
 636
 637         fm10k_mbx_lock(hw);
 638         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 639                                 mode);
 640         fm10k_mbx_unlock(hw);
 641
 642         if (status != FM10K_SUCCESS)
 643                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 644 }
 645
 646 static void
 647 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 648 {
 649         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 650         int status;
 651
 652         PMD_INIT_FUNC_TRACE();
 653
 654         /* Return if it didn't acquire valid glort range */
 655         if (!fm10k_glort_valid(hw))
 656                 return;
 657
 658         /* If promiscuous mode is enabled, it doesn't make sense to enable
 659          * allmulticast and disable promiscuous since fm10k only can select
 660          * one of the modes.
 661          */
 662         if (dev->data->promiscuous) {
 663                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 664                         "needn't enable allmulticast");
 665                 return;
 666         }
 667
 668         fm10k_mbx_lock(hw);
 669         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 670                                 FM10K_XCAST_MODE_ALLMULTI);
 671         fm10k_mbx_unlock(hw);
 672
 673         if (status != FM10K_SUCCESS)
 674                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 675 }
 676
 677 static void
 678 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 679 {
 680         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 681         int status;
 682
 683         PMD_INIT_FUNC_TRACE();
 684
 685         /* Return if it didn't acquire valid glort range */
 686         if (!fm10k_glort_valid(hw))
 687                 return;
 688
 689         if (dev->data->promiscuous) {
 690                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 691                         "since promisc mode is enabled");
 692                 return;
 693         }
 694
 695         fm10k_mbx_lock(hw);
 696         /* Change mode to unicast mode */
 697         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 698                                 FM10K_XCAST_MODE_NONE);
 699         fm10k_mbx_unlock(hw);
 700
 701         if (status != FM10K_SUCCESS)
 702                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 703 }
 704
 705 /* fls = find last set bit = 32 minus the number of leading zeros */
 706 #ifndef fls
 707 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 708 #endif
 709 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 710 static int
 711 fm10k_dev_start(struct rte_eth_dev *dev)
 712 {
 713         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 714         int i, diag;
 715
 716         PMD_INIT_FUNC_TRACE();
 717
 718         /* stop, init, then start the hw */
 719         diag = fm10k_stop_hw(hw);
 720         if (diag != FM10K_SUCCESS) {
 721                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 722                 return -EIO;
 723         }
 724
 725         diag = fm10k_init_hw(hw);
 726         if (diag != FM10K_SUCCESS) {
 727                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 728                 return -EIO;
 729         }
 730
 731         diag = fm10k_start_hw(hw);
 732         if (diag != FM10K_SUCCESS) {
 733                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 734                 return -EIO;
 735         }
 736
 737         diag = fm10k_dev_tx_init(dev);
 738         if (diag) {
 739                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 740                 return diag;
 741         }
 742
 743         diag = fm10k_dev_rx_init(dev);
 744         if (diag) {
 745                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 746                 return diag;
 747         }
 748
 749         if (hw->mac.type == fm10k_mac_pf) {
 750                 /* Establish only VSI 0 as valid */
 751                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 752
 753                 /* Configure RSS bits used in RETA table */
 754                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 755                                 fls(dev->data->nb_rx_queues - 1) <<
 756                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 757
 758                 /* Invalidate all other GLORT entries */
 759                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 760                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 761                                         FM10K_DGLORTMAP_NONE);
 762         }
 763
 764         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 765                 struct fm10k_rx_queue *rxq;
 766                 rxq = dev->data->rx_queues[i];
 767
 768                 if (rxq->rx_deferred_start)
 769                         continue;
 770                 diag = fm10k_dev_rx_queue_start(dev, i);
 771                 if (diag != 0) {
 772                         int j;
 773                         for (j = 0; j < i; ++j)
 774                                 rx_queue_clean(dev->data->rx_queues[j]);
 775                         return diag;
 776                 }
 777         }
 778
 779         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 780                 struct fm10k_tx_queue *txq;
 781                 txq = dev->data->tx_queues[i];
 782
 783                 if (txq->tx_deferred_start)
 784                         continue;
 785                 diag = fm10k_dev_tx_queue_start(dev, i);
 786                 if (diag != 0) {
 787                         int j;
 788                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 789                                 rx_queue_clean(dev->data->rx_queues[j]);
 790                         return diag;
 791                 }
 792         }
 793
 794         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID) {
 795                 /* Update default vlan */
 796                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 797
 798                 /* Add default mac/vlan filter to PF/Switch manager */
 799                 fm10k_MAC_filter_set(dev, hw->mac.addr, true);
 800         }
 801
 802         return 0;
 803 }
 804
 805 static void
 806 fm10k_dev_stop(struct rte_eth_dev *dev)
 807 {
 808         int i;
 809
 810         PMD_INIT_FUNC_TRACE();
 811
 812         for (i = 0; i < dev->data->nb_tx_queues; i++)
 813                 fm10k_dev_tx_queue_stop(dev, i);
 814
 815         for (i = 0; i < dev->data->nb_rx_queues; i++)
 816                 fm10k_dev_rx_queue_stop(dev, i);
 817 }
 818
 819 static void
 820 fm10k_dev_close(struct rte_eth_dev *dev)
 821 {
 822         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 823
 824         PMD_INIT_FUNC_TRACE();
 825
 826         fm10k_MACVLAN_remove_all(dev);
 827
 828         /* Stop mailbox service first */
 829         fm10k_close_mbx_service(hw);
 830         fm10k_dev_stop(dev);
 831         fm10k_stop_hw(hw);
 832 }
 833
 834 static int
 835 fm10k_link_update(struct rte_eth_dev *dev,
 836         __rte_unused int wait_to_complete)
 837 {
 838         PMD_INIT_FUNC_TRACE();
 839
 840         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 841          * x8 PCIe interface. For now, we leave the speed undefined since there
 842          * is no 50Gbps Ethernet. */
 843         dev->data->dev_link.link_speed  = 0;
 844         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 845         dev->data->dev_link.link_status = 1;
 846
 847         return 0;
 848 }
 849
 850 static void
 851 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 852 {
 853         uint64_t ipackets, opackets, ibytes, obytes;
 854         struct fm10k_hw *hw =
 855                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 856         struct fm10k_hw_stats *hw_stats =
 857                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 858         int i;
 859
 860         PMD_INIT_FUNC_TRACE();
 861
 862         fm10k_update_hw_stats(hw, hw_stats);
 863
 864         ipackets = opackets = ibytes = obytes = 0;
 865         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 866                 (i < hw->mac.max_queues); ++i) {
 867                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 868                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 869                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 870                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 871                 ipackets += stats->q_ipackets[i];
 872                 opackets += stats->q_opackets[i];
 873                 ibytes   += stats->q_ibytes[i];
 874                 obytes   += stats->q_obytes[i];
 875         }
 876         stats->ipackets = ipackets;
 877         stats->opackets = opackets;
 878         stats->ibytes = ibytes;
 879         stats->obytes = obytes;
 880 }
 881
 882 static void
 883 fm10k_stats_reset(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886         struct fm10k_hw_stats *hw_stats =
 887                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 888
 889         PMD_INIT_FUNC_TRACE();
 890
 891         memset(hw_stats, 0, sizeof(*hw_stats));
 892         fm10k_rebind_hw_stats(hw, hw_stats);
 893 }
 894
 895 static void
 896 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 897         struct rte_eth_dev_info *dev_info)
 898 {
 899         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 900
 901         PMD_INIT_FUNC_TRACE();
 902
 903         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 904         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 905         dev_info->max_rx_queues      = hw->mac.max_queues;
 906         dev_info->max_tx_queues      = hw->mac.max_queues;
 907         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
 908         dev_info->max_hash_mac_addrs = 0;
 909         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 910         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 911         dev_info->rx_offload_capa =
 912                 DEV_RX_OFFLOAD_VLAN_STRIP |
 913                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 914                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 915                 DEV_RX_OFFLOAD_TCP_CKSUM;
 916         dev_info->tx_offload_capa =
 917                 DEV_TX_OFFLOAD_VLAN_INSERT;
 918         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 919
 920         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 921                 .rx_thresh = {
 922                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 923                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 924                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 925                 },
 926                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 927                 .rx_drop_en = 0,
 928         };
 929
 930         dev_info->default_txconf = (struct rte_eth_txconf) {
 931                 .tx_thresh = {
 932                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 933                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 934                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 935                 },
 936                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 937                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 938                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 939                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 940         };
 941
 942 }
 943
 944 static int
 945 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 946 {
 947         s32 result;
 948         uint16_t mac_num = 0;
 949         uint32_t vid_idx, vid_bit, mac_index;
 950         struct fm10k_hw *hw;
 951         struct fm10k_macvlan_filter_info *macvlan;
 952         struct rte_eth_dev_data *data = dev->data;
 953
 954         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 955         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 956
 957         if (vlan_id > ETH_VLAN_ID_MAX) {
 958                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
 959                 return (-EINVAL);
 960         }
 961
 962         vid_idx = FM10K_VFTA_IDX(vlan_id);
 963         vid_bit = FM10K_VFTA_BIT(vlan_id);
 964         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
 965         if (on && (macvlan->vfta[vid_idx] & vid_bit))
 966                 return 0;
 967         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
 968         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
 969                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
 970                         "in the VLAN filter table");
 971                 return (-EINVAL);
 972         }
 973
 974         fm10k_mbx_lock(hw);
 975         result = fm10k_update_vlan(hw, vlan_id, 0, on);
 976         fm10k_mbx_unlock(hw);
 977         if (result != FM10K_SUCCESS) {
 978                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
 979                 return (-EIO);
 980         }
 981
 982         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
 983                         (result == FM10K_SUCCESS); mac_index++) {
 984                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
 985                         continue;
 986                 if (mac_num > macvlan->mac_num - 1) {
 987                         PMD_INIT_LOG(ERR, "MAC address number "
 988                                         "not match");
 989                         break;
 990                 }
 991                 fm10k_mbx_lock(hw);
 992                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
 993                         data->mac_addrs[mac_index].addr_bytes,
 994                         vlan_id, on, 0);
 995                 fm10k_mbx_unlock(hw);
 996                 mac_num++;
 997         }
 998         if (result != FM10K_SUCCESS) {
 999                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1000                 return (-EIO);
1001         }
1002
1003         if (on) {
1004                 macvlan->vlan_num++;
1005                 macvlan->vfta[vid_idx] |= vid_bit;
1006         } else {
1007                 macvlan->vlan_num--;
1008                 macvlan->vfta[vid_idx] &= ~vid_bit;
1009         }
1010         return 0;
1011 }
1012
1013 static void
1014 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1015 {
1016         if (mask & ETH_VLAN_STRIP_MASK) {
1017                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1018                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1019                                         "always on in fm10k");
1020         }
1021
1022         if (mask & ETH_VLAN_EXTEND_MASK) {
1023                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1024                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1025                                         "supported in fm10k");
1026         }
1027
1028         if (mask & ETH_VLAN_FILTER_MASK) {
1029                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1030                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1031         }
1032 }
1033
1034 /* Add/Remove a MAC address, and update filters */
1035 static void
1036 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add)
1037 {
1038         uint32_t i, j, k;
1039         struct fm10k_hw *hw;
1040         struct fm10k_macvlan_filter_info *macvlan;
1041
1042         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1043         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1044
1045         i = 0;
1046         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1047                 if (macvlan->vfta[j]) {
1048                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1049                                 if (macvlan->vfta[j] & (1 << k)) {
1050                                         if (i + 1 > macvlan->vlan_num) {
1051                                                 PMD_INIT_LOG(ERR, "vlan number "
1052                                                                 "not match");
1053                                                 return;
1054                                         }
1055                                         fm10k_mbx_lock(hw);
1056                                         fm10k_update_uc_addr(hw,
1057                                                 hw->mac.dglort_map, mac,
1058                                                 j * FM10K_UINT32_BIT_SIZE + k,
1059                                                 add, 0);
1060                                         fm10k_mbx_unlock(hw);
1061                                         i++;
1062                                 }
1063                         }
1064                 }
1065         }
1066
1067         if (add)
1068                 macvlan->mac_num++;
1069         else
1070                 macvlan->mac_num--;
1071 }
1072
1073 /* Add a MAC address, and update filters */
1074 static void
1075 fm10k_macaddr_add(struct rte_eth_dev *dev,
1076                  struct ether_addr *mac_addr,
1077                  __rte_unused uint32_t index,
1078                  __rte_unused uint32_t pool)
1079 {
1080         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE);
1081 }
1082
1083 /* Remove a MAC address, and update filters */
1084 static void
1085 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1086 {
1087         struct rte_eth_dev_data *data = dev->data;
1088
1089         if (index < FM10K_MAX_MACADDR_NUM)
1090                 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1091                                 FALSE);
1092 }
1093
1094 /* Remove all VLAN and MAC address table entries */
1095 static void
1096 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev)
1097 {
1098         uint32_t j, k;
1099         struct fm10k_macvlan_filter_info *macvlan;
1100
1101         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1102         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1103                 if (macvlan->vfta[j]) {
1104                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1105                                 if (macvlan->vfta[j] & (1 << k))
1106                                         fm10k_vlan_filter_set(dev,
1107                                                 j * FM10K_UINT32_BIT_SIZE + k, false);
1108                         }
1109                 }
1110         }
1111 }
1112
1113 static inline int
1114 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1115 {
1116         if ((request < min) || (request > max) || ((request % mult) != 0))
1117                 return -1;
1118         else
1119                 return 0;
1120 }
1121
1122 /*
1123  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1124  * the physical address is required. If the memzone is already created, then
1125  * this function returns a pointer to the existing memzone.
1126  */
1127 static inline const struct rte_memzone *
1128 allocate_hw_ring(const char *driver_name, const char *ring_name,
1129         uint8_t port_id, uint16_t queue_id, int socket_id,
1130         uint32_t size, uint32_t align)
1131 {
1132         char name[RTE_MEMZONE_NAMESIZE];
1133         const struct rte_memzone *mz;
1134
1135         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1136                  driver_name, ring_name, port_id, queue_id, socket_id);
1137
1138         /* return the memzone if it already exists */
1139         mz = rte_memzone_lookup(name);
1140         if (mz)
1141                 return mz;
1142
1143 #ifdef RTE_LIBRTE_XEN_DOM0
1144         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1145                                            RTE_PGSIZE_2M);
1146 #else
1147         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1148 #endif
1149 }
1150
1151 static inline int
1152 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1153 {
1154         if ((request < min) || (request > max) || ((div % request) != 0))
1155                 return -1;
1156         else
1157                 return 0;
1158 }
1159
1160 static inline int
1161 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1162 {
1163         uint16_t rx_free_thresh;
1164
1165         if (conf->rx_free_thresh == 0)
1166                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1167         else
1168                 rx_free_thresh = conf->rx_free_thresh;
1169
1170         /* make sure the requested threshold satisfies the constraints */
1171         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1172                         FM10K_RX_FREE_THRESH_MAX(q),
1173                         FM10K_RX_FREE_THRESH_DIV(q),
1174                         rx_free_thresh)) {
1175                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1176                         "less than or equal to %u, "
1177                         "greater than or equal to %u, "
1178                         "and a divisor of %u",
1179                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1180                         FM10K_RX_FREE_THRESH_MIN(q),
1181                         FM10K_RX_FREE_THRESH_DIV(q));
1182                 return (-EINVAL);
1183         }
1184
1185         q->alloc_thresh = rx_free_thresh;
1186         q->drop_en = conf->rx_drop_en;
1187         q->rx_deferred_start = conf->rx_deferred_start;
1188
1189         return 0;
1190 }
1191
1192 /*
1193  * Hardware requires specific alignment for Rx packet buffers. At
1194  * least one of the following two conditions must be satisfied.
1195  *  1. Address is 512B aligned
1196  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1197  *
1198  * As such, the driver may need to adjust the DMA address within the
1199  * buffer by up to 512B.
1200  *
1201  * return 1 if the element size is valid, otherwise return 0.
1202  */
1203 static int
1204 mempool_element_size_valid(struct rte_mempool *mp)
1205 {
1206         uint32_t min_size;
1207
1208         /* elt_size includes mbuf header and headroom */
1209         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1210                         RTE_PKTMBUF_HEADROOM;
1211
1212         /* account for up to 512B of alignment */
1213         min_size -= FM10K_RX_DATABUF_ALIGN;
1214
1215         /* sanity check for overflow */
1216         if (min_size > mp->elt_size)
1217                 return 0;
1218
1219         /* size is valid */
1220         return 1;
1221 }
1222
1223 static int
1224 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1225         uint16_t nb_desc, unsigned int socket_id,
1226         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1227 {
1228         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1229         struct fm10k_rx_queue *q;
1230         const struct rte_memzone *mz;
1231
1232         PMD_INIT_FUNC_TRACE();
1233
1234         /* make sure the mempool element size can account for alignment. */
1235         if (!mempool_element_size_valid(mp)) {
1236                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1237                 return (-EINVAL);
1238         }
1239
1240         /* make sure a valid number of descriptors have been requested */
1241         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1242                                 FM10K_MULT_RX_DESC, nb_desc)) {
1243                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1244                         "less than or equal to %"PRIu32", "
1245                         "greater than or equal to %u, "
1246                         "and a multiple of %u",
1247                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1248                         FM10K_MULT_RX_DESC);
1249                 return (-EINVAL);
1250         }
1251
1252         /*
1253          * if this queue existed already, free the associated memory. The
1254          * queue cannot be reused in case we need to allocate memory on
1255          * different socket than was previously used.
1256          */
1257         if (dev->data->rx_queues[queue_id] != NULL) {
1258                 rx_queue_free(dev->data->rx_queues[queue_id]);
1259                 dev->data->rx_queues[queue_id] = NULL;
1260         }
1261
1262         /* allocate memory for the queue structure */
1263         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1264                                 socket_id);
1265         if (q == NULL) {
1266                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1267                 return (-ENOMEM);
1268         }
1269
1270         /* setup queue */
1271         q->mp = mp;
1272         q->nb_desc = nb_desc;
1273         q->port_id = dev->data->port_id;
1274         q->queue_id = queue_id;
1275         q->tail_ptr = (volatile uint32_t *)
1276                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1277         if (handle_rxconf(q, conf))
1278                 return (-EINVAL);
1279
1280         /* allocate memory for the software ring */
1281         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1282                                         nb_desc * sizeof(struct rte_mbuf *),
1283                                         RTE_CACHE_LINE_SIZE, socket_id);
1284         if (q->sw_ring == NULL) {
1285                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1286                 rte_free(q);
1287                 return (-ENOMEM);
1288         }
1289
1290         /*
1291          * allocate memory for the hardware descriptor ring. A memzone large
1292          * enough to hold the maximum ring size is requested to allow for
1293          * resizing in later calls to the queue setup function.
1294          */
1295         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1296                                 dev->data->port_id, queue_id, socket_id,
1297                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1298         if (mz == NULL) {
1299                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1300                 rte_free(q->sw_ring);
1301                 rte_free(q);
1302                 return (-ENOMEM);
1303         }
1304         q->hw_ring = mz->addr;
1305         q->hw_ring_phys_addr = mz->phys_addr;
1306
1307         dev->data->rx_queues[queue_id] = q;
1308         return 0;
1309 }
1310
1311 static void
1312 fm10k_rx_queue_release(void *queue)
1313 {
1314         PMD_INIT_FUNC_TRACE();
1315
1316         rx_queue_free(queue);
1317 }
1318
1319 static inline int
1320 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1321 {
1322         uint16_t tx_free_thresh;
1323         uint16_t tx_rs_thresh;
1324
1325         /* constraint MACROs require that tx_free_thresh is configured
1326          * before tx_rs_thresh */
1327         if (conf->tx_free_thresh == 0)
1328                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1329         else
1330                 tx_free_thresh = conf->tx_free_thresh;
1331
1332         /* make sure the requested threshold satisfies the constraints */
1333         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1334                         FM10K_TX_FREE_THRESH_MAX(q),
1335                         FM10K_TX_FREE_THRESH_DIV(q),
1336                         tx_free_thresh)) {
1337                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1338                         "less than or equal to %u, "
1339                         "greater than or equal to %u, "
1340                         "and a divisor of %u",
1341                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1342                         FM10K_TX_FREE_THRESH_MIN(q),
1343                         FM10K_TX_FREE_THRESH_DIV(q));
1344                 return (-EINVAL);
1345         }
1346
1347         q->free_thresh = tx_free_thresh;
1348
1349         if (conf->tx_rs_thresh == 0)
1350                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1351         else
1352                 tx_rs_thresh = conf->tx_rs_thresh;
1353
1354         q->tx_deferred_start = conf->tx_deferred_start;
1355
1356         /* make sure the requested threshold satisfies the constraints */
1357         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1358                         FM10K_TX_RS_THRESH_MAX(q),
1359                         FM10K_TX_RS_THRESH_DIV(q),
1360                         tx_rs_thresh)) {
1361                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1362                         "less than or equal to %u, "
1363                         "greater than or equal to %u, "
1364                         "and a divisor of %u",
1365                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1366                         FM10K_TX_RS_THRESH_MIN(q),
1367                         FM10K_TX_RS_THRESH_DIV(q));
1368                 return (-EINVAL);
1369         }
1370
1371         q->rs_thresh = tx_rs_thresh;
1372
1373         return 0;
1374 }
1375
1376 static int
1377 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1378         uint16_t nb_desc, unsigned int socket_id,
1379         const struct rte_eth_txconf *conf)
1380 {
1381         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1382         struct fm10k_tx_queue *q;
1383         const struct rte_memzone *mz;
1384
1385         PMD_INIT_FUNC_TRACE();
1386
1387         /* make sure a valid number of descriptors have been requested */
1388         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1389                                 FM10K_MULT_TX_DESC, nb_desc)) {
1390                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1391                         "less than or equal to %"PRIu32", "
1392                         "greater than or equal to %u, "
1393                         "and a multiple of %u",
1394                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1395                         FM10K_MULT_TX_DESC);
1396                 return (-EINVAL);
1397         }
1398
1399         /*
1400          * if this queue existed already, free the associated memory. The
1401          * queue cannot be reused in case we need to allocate memory on
1402          * different socket than was previously used.
1403          */
1404         if (dev->data->tx_queues[queue_id] != NULL) {
1405                 tx_queue_free(dev->data->tx_queues[queue_id]);
1406                 dev->data->tx_queues[queue_id] = NULL;
1407         }
1408
1409         /* allocate memory for the queue structure */
1410         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1411                                 socket_id);
1412         if (q == NULL) {
1413                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1414                 return (-ENOMEM);
1415         }
1416
1417         /* setup queue */
1418         q->nb_desc = nb_desc;
1419         q->port_id = dev->data->port_id;
1420         q->queue_id = queue_id;
1421         q->tail_ptr = (volatile uint32_t *)
1422                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1423         if (handle_txconf(q, conf))
1424                 return (-EINVAL);
1425
1426         /* allocate memory for the software ring */
1427         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1428                                         nb_desc * sizeof(struct rte_mbuf *),
1429                                         RTE_CACHE_LINE_SIZE, socket_id);
1430         if (q->sw_ring == NULL) {
1431                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1432                 rte_free(q);
1433                 return (-ENOMEM);
1434         }
1435
1436         /*
1437          * allocate memory for the hardware descriptor ring. A memzone large
1438          * enough to hold the maximum ring size is requested to allow for
1439          * resizing in later calls to the queue setup function.
1440          */
1441         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1442                                 dev->data->port_id, queue_id, socket_id,
1443                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1444         if (mz == NULL) {
1445                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1446                 rte_free(q->sw_ring);
1447                 rte_free(q);
1448                 return (-ENOMEM);
1449         }
1450         q->hw_ring = mz->addr;
1451         q->hw_ring_phys_addr = mz->phys_addr;
1452
1453         /*
1454          * allocate memory for the RS bit tracker. Enough slots to hold the
1455          * descriptor index for each RS bit needing to be set are required.
1456          */
1457         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1458                                 ((nb_desc + 1) / q->rs_thresh) *
1459                                 sizeof(uint16_t),
1460                                 RTE_CACHE_LINE_SIZE, socket_id);
1461         if (q->rs_tracker.list == NULL) {
1462                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1463                 rte_free(q->sw_ring);
1464                 rte_free(q);
1465                 return (-ENOMEM);
1466         }
1467
1468         dev->data->tx_queues[queue_id] = q;
1469         return 0;
1470 }
1471
1472 static void
1473 fm10k_tx_queue_release(void *queue)
1474 {
1475         PMD_INIT_FUNC_TRACE();
1476
1477         tx_queue_free(queue);
1478 }
1479
1480 static int
1481 fm10k_reta_update(struct rte_eth_dev *dev,
1482                         struct rte_eth_rss_reta_entry64 *reta_conf,
1483                         uint16_t reta_size)
1484 {
1485         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1486         uint16_t i, j, idx, shift;
1487         uint8_t mask;
1488         uint32_t reta;
1489
1490         PMD_INIT_FUNC_TRACE();
1491
1492         if (reta_size > FM10K_MAX_RSS_INDICES) {
1493                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1494                         "(%d) doesn't match the number hardware can supported "
1495                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1496                 return -EINVAL;
1497         }
1498
1499         /*
1500          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1501          * 128-entries in 32 registers
1502          */
1503         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1504                 idx = i / RTE_RETA_GROUP_SIZE;
1505                 shift = i % RTE_RETA_GROUP_SIZE;
1506                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1507                                 BIT_MASK_PER_UINT32);
1508                 if (mask == 0)
1509                         continue;
1510
1511                 reta = 0;
1512                 if (mask != BIT_MASK_PER_UINT32)
1513                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1514
1515                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1516                         if (mask & (0x1 << j)) {
1517                                 if (mask != 0xF)
1518                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1519                                 reta |= reta_conf[idx].reta[shift + j] <<
1520                                                 (CHAR_BIT * j);
1521                         }
1522                 }
1523                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1524         }
1525
1526         return 0;
1527 }
1528
1529 static int
1530 fm10k_reta_query(struct rte_eth_dev *dev,
1531                         struct rte_eth_rss_reta_entry64 *reta_conf,
1532                         uint16_t reta_size)
1533 {
1534         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1535         uint16_t i, j, idx, shift;
1536         uint8_t mask;
1537         uint32_t reta;
1538
1539         PMD_INIT_FUNC_TRACE();
1540
1541         if (reta_size < FM10K_MAX_RSS_INDICES) {
1542                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1543                         "(%d) doesn't match the number hardware can supported "
1544                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1545                 return -EINVAL;
1546         }
1547
1548         /*
1549          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1550          * 128-entries in 32 registers
1551          */
1552         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1553                 idx = i / RTE_RETA_GROUP_SIZE;
1554                 shift = i % RTE_RETA_GROUP_SIZE;
1555                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1556                                 BIT_MASK_PER_UINT32);
1557                 if (mask == 0)
1558                         continue;
1559
1560                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1561                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1562                         if (mask & (0x1 << j))
1563                                 reta_conf[idx].reta[shift + j] = ((reta >>
1564                                         CHAR_BIT * j) & UINT8_MAX);
1565                 }
1566         }
1567
1568         return 0;
1569 }
1570
1571 static int
1572 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1573         struct rte_eth_rss_conf *rss_conf)
1574 {
1575         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1576         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1577         uint32_t mrqc;
1578         uint64_t hf = rss_conf->rss_hf;
1579         int i;
1580
1581         PMD_INIT_FUNC_TRACE();
1582
1583         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1584                 FM10K_RSSRK_ENTRIES_PER_REG)
1585                 return -EINVAL;
1586
1587         if (hf == 0)
1588                 return -EINVAL;
1589
1590         mrqc = 0;
1591         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1592         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1593         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1594         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1595         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1596         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1597         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1598         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1599         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1600
1601         /* If the mapping doesn't fit any supported, return */
1602         if (mrqc == 0)
1603                 return -EINVAL;
1604
1605         if (key != NULL)
1606                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1607                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1608
1609         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1610
1611         return 0;
1612 }
1613
1614 static int
1615 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1616         struct rte_eth_rss_conf *rss_conf)
1617 {
1618         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1619         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1620         uint32_t mrqc;
1621         uint64_t hf;
1622         int i;
1623
1624         PMD_INIT_FUNC_TRACE();
1625
1626         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1627                                 FM10K_RSSRK_ENTRIES_PER_REG)
1628                 return -EINVAL;
1629
1630         if (key != NULL)
1631                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1632                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1633
1634         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1635         hf = 0;
1636         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1637         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1638         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1639         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1640         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1641         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1642         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1643         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1644         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1645
1646         rss_conf->rss_hf = hf;
1647
1648         return 0;
1649 }
1650
1651 static void
1652 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1653 {
1654         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1655         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1656
1657         /* Bind all local non-queue interrupt to vector 0 */
1658         int_map |= 0;
1659
1660         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1661         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1662         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1663         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1664         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1665         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1666
1667         /* Enable misc causes */
1668         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1669                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1670                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1671                                 FM10K_EIMR_ENABLE(MAILBOX) |
1672                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1673                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1674                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1675                                 FM10K_EIMR_ENABLE(VFLR));
1676
1677         /* Enable ITR 0 */
1678         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1679                                         FM10K_ITR_MASK_CLEAR);
1680         FM10K_WRITE_FLUSH(hw);
1681 }
1682
1683 static void
1684 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1685 {
1686         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1687         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1688
1689         /* Bind all local non-queue interrupt to vector 0 */
1690         int_map |= 0;
1691
1692         /* Only INT 0 available, other 15 are reserved. */
1693         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1694
1695         /* Enable ITR 0 */
1696         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1697                                         FM10K_ITR_MASK_CLEAR);
1698         FM10K_WRITE_FLUSH(hw);
1699 }
1700
1701 static int
1702 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1703 {
1704         struct fm10k_fault fault;
1705         int err;
1706         const char *estr = "Unknown error";
1707
1708         /* Process PCA fault */
1709         if (eicr & FM10K_EIMR_PCA_FAULT) {
1710                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1711                 if (err)
1712                         goto error;
1713                 switch (fault.type) {
1714                 case PCA_NO_FAULT:
1715                         estr = "PCA_NO_FAULT"; break;
1716                 case PCA_UNMAPPED_ADDR:
1717                         estr = "PCA_UNMAPPED_ADDR"; break;
1718                 case PCA_BAD_QACCESS_PF:
1719                         estr = "PCA_BAD_QACCESS_PF"; break;
1720                 case PCA_BAD_QACCESS_VF:
1721                         estr = "PCA_BAD_QACCESS_VF"; break;
1722                 case PCA_MALICIOUS_REQ:
1723                         estr = "PCA_MALICIOUS_REQ"; break;
1724                 case PCA_POISONED_TLP:
1725                         estr = "PCA_POISONED_TLP"; break;
1726                 case PCA_TLP_ABORT:
1727                         estr = "PCA_TLP_ABORT"; break;
1728                 default:
1729                         goto error;
1730                 }
1731                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1732                         estr, fault.func ? "VF" : "PF", fault.func,
1733                         fault.address, fault.specinfo);
1734         }
1735
1736         /* Process THI fault */
1737         if (eicr & FM10K_EIMR_THI_FAULT) {
1738                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1739                 if (err)
1740                         goto error;
1741                 switch (fault.type) {
1742                 case THI_NO_FAULT:
1743                         estr = "THI_NO_FAULT"; break;
1744                 case THI_MAL_DIS_Q_FAULT:
1745                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1746                 default:
1747                         goto error;
1748                 }
1749                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1750                         estr, fault.func ? "VF" : "PF", fault.func,
1751                         fault.address, fault.specinfo);
1752         }
1753
1754         /* Process FUM fault */
1755         if (eicr & FM10K_EIMR_FUM_FAULT) {
1756                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1757                 if (err)
1758                         goto error;
1759                 switch (fault.type) {
1760                 case FUM_NO_FAULT:
1761                         estr = "FUM_NO_FAULT"; break;
1762                 case FUM_UNMAPPED_ADDR:
1763                         estr = "FUM_UNMAPPED_ADDR"; break;
1764                 case FUM_POISONED_TLP:
1765                         estr = "FUM_POISONED_TLP"; break;
1766                 case FUM_BAD_VF_QACCESS:
1767                         estr = "FUM_BAD_VF_QACCESS"; break;
1768                 case FUM_ADD_DECODE_ERR:
1769                         estr = "FUM_ADD_DECODE_ERR"; break;
1770                 case FUM_RO_ERROR:
1771                         estr = "FUM_RO_ERROR"; break;
1772                 case FUM_QPRC_CRC_ERROR:
1773                         estr = "FUM_QPRC_CRC_ERROR"; break;
1774                 case FUM_CSR_TIMEOUT:
1775                         estr = "FUM_CSR_TIMEOUT"; break;
1776                 case FUM_INVALID_TYPE:
1777                         estr = "FUM_INVALID_TYPE"; break;
1778                 case FUM_INVALID_LENGTH:
1779                         estr = "FUM_INVALID_LENGTH"; break;
1780                 case FUM_INVALID_BE:
1781                         estr = "FUM_INVALID_BE"; break;
1782                 case FUM_INVALID_ALIGN:
1783                         estr = "FUM_INVALID_ALIGN"; break;
1784                 default:
1785                         goto error;
1786                 }
1787                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1788                         estr, fault.func ? "VF" : "PF", fault.func,
1789                         fault.address, fault.specinfo);
1790         }
1791
1792         if (estr)
1793                 return 0;
1794         return 0;
1795 error:
1796         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1797         return err;
1798 }
1799
1800 /**
1801  * PF interrupt handler triggered by NIC for handling specific interrupt.
1802  *
1803  * @param handle
1804  *  Pointer to interrupt handle.
1805  * @param param
1806  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1807  *
1808  * @return
1809  *  void
1810  */
1811 static void
1812 fm10k_dev_interrupt_handler_pf(
1813                         __rte_unused struct rte_intr_handle *handle,
1814                         void *param)
1815 {
1816         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1817         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1818         uint32_t cause, status;
1819
1820         if (hw->mac.type != fm10k_mac_pf)
1821                 return;
1822
1823         cause = FM10K_READ_REG(hw, FM10K_EICR);
1824
1825         /* Handle PCI fault cases */
1826         if (cause & FM10K_EICR_FAULT_MASK) {
1827                 PMD_INIT_LOG(ERR, "INT: find fault!");
1828                 fm10k_dev_handle_fault(hw, cause);
1829         }
1830
1831         /* Handle switch up/down */
1832         if (cause & FM10K_EICR_SWITCHNOTREADY)
1833                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1834
1835         if (cause & FM10K_EICR_SWITCHREADY)
1836                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1837
1838         /* Handle mailbox message */
1839         fm10k_mbx_lock(hw);
1840         hw->mbx.ops.process(hw, &hw->mbx);
1841         fm10k_mbx_unlock(hw);
1842
1843         /* Handle SRAM error */
1844         if (cause & FM10K_EICR_SRAMERROR) {
1845                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1846
1847                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1848                 /* Write to clear pending bits */
1849                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1850
1851                 /* Todo: print out error message after shared code  updates */
1852         }
1853
1854         /* Clear these 3 events if having any */
1855         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1856                  FM10K_EICR_SWITCHREADY;
1857         if (cause)
1858                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1859
1860         /* Re-enable interrupt from device side */
1861         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1862                                         FM10K_ITR_MASK_CLEAR);
1863         /* Re-enable interrupt from host side */
1864         rte_intr_enable(&(dev->pci_dev->intr_handle));
1865 }
1866
1867 /**
1868  * VF interrupt handler triggered by NIC for handling specific interrupt.
1869  *
1870  * @param handle
1871  *  Pointer to interrupt handle.
1872  * @param param
1873  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1874  *
1875  * @return
1876  *  void
1877  */
1878 static void
1879 fm10k_dev_interrupt_handler_vf(
1880                         __rte_unused struct rte_intr_handle *handle,
1881                         void *param)
1882 {
1883         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1884         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1885
1886         if (hw->mac.type != fm10k_mac_vf)
1887                 return;
1888
1889         /* Handle mailbox message if lock is acquired */
1890         fm10k_mbx_lock(hw);
1891         hw->mbx.ops.process(hw, &hw->mbx);
1892         fm10k_mbx_unlock(hw);
1893
1894         /* Re-enable interrupt from device side */
1895         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1896                                         FM10K_ITR_MASK_CLEAR);
1897         /* Re-enable interrupt from host side */
1898         rte_intr_enable(&(dev->pci_dev->intr_handle));
1899 }
1900
1901 /* Mailbox message handler in VF */
1902 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1903         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1904         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1905         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1906         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1907 };
1908
1909 /* Mailbox message handler in PF */
1910 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1911         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1912         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1913         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1914         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1915         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1916         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1917         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1918 };
1919
1920 static int
1921 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1922 {
1923         int err;
1924
1925         /* Initialize mailbox lock */
1926         fm10k_mbx_initlock(hw);
1927
1928         /* Replace default message handler with new ones */
1929         if (hw->mac.type == fm10k_mac_pf)
1930                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1931         else
1932                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1933
1934         if (err) {
1935                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1936                                 err);
1937                 return err;
1938         }
1939         /* Connect to SM for PF device or PF for VF device */
1940         return hw->mbx.ops.connect(hw, &hw->mbx);
1941 }
1942
1943 static void
1944 fm10k_close_mbx_service(struct fm10k_hw *hw)
1945 {
1946         /* Disconnect from SM for PF device or PF for VF device */
1947         hw->mbx.ops.disconnect(hw, &hw->mbx);
1948 }
1949
1950 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1951         .dev_configure          = fm10k_dev_configure,
1952         .dev_start              = fm10k_dev_start,
1953         .dev_stop               = fm10k_dev_stop,
1954         .dev_close              = fm10k_dev_close,
1955         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
1956         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
1957         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
1958         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
1959         .stats_get              = fm10k_stats_get,
1960         .stats_reset            = fm10k_stats_reset,
1961         .link_update            = fm10k_link_update,
1962         .dev_infos_get          = fm10k_dev_infos_get,
1963         .vlan_filter_set        = fm10k_vlan_filter_set,
1964         .vlan_offload_set       = fm10k_vlan_offload_set,
1965         .mac_addr_add           = fm10k_macaddr_add,
1966         .mac_addr_remove        = fm10k_macaddr_remove,
1967         .rx_queue_start         = fm10k_dev_rx_queue_start,
1968         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1969         .tx_queue_start         = fm10k_dev_tx_queue_start,
1970         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1971         .rx_queue_setup         = fm10k_rx_queue_setup,
1972         .rx_queue_release       = fm10k_rx_queue_release,
1973         .tx_queue_setup         = fm10k_tx_queue_setup,
1974         .tx_queue_release       = fm10k_tx_queue_release,
1975         .reta_update            = fm10k_reta_update,
1976         .reta_query             = fm10k_reta_query,
1977         .rss_hash_update        = fm10k_rss_hash_update,
1978         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1979 };
1980
1981 static int
1982 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1983 {
1984         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1985         int diag;
1986         struct fm10k_macvlan_filter_info *macvlan;
1987
1988         PMD_INIT_FUNC_TRACE();
1989
1990         dev->dev_ops = &fm10k_eth_dev_ops;
1991         dev->rx_pkt_burst = &fm10k_recv_pkts;
1992         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1993
1994         if (dev->data->scattered_rx)
1995                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1996
1997         /* only initialize in the primary process */
1998         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1999                 return 0;
2000
2001         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2002         memset(macvlan, 0, sizeof(*macvlan));
2003         /* Vendor and Device ID need to be set before init of shared code */
2004         memset(hw, 0, sizeof(*hw));
2005         hw->device_id = dev->pci_dev->id.device_id;
2006         hw->vendor_id = dev->pci_dev->id.vendor_id;
2007         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2008         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2009         hw->revision_id = 0;
2010         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2011         if (hw->hw_addr == NULL) {
2012                 PMD_INIT_LOG(ERR, "Bad mem resource."
2013                         " Try to blacklist unused devices.");
2014                 return -EIO;
2015         }
2016
2017         /* Store fm10k_adapter pointer */
2018         hw->back = dev->data->dev_private;
2019
2020         /* Initialize the shared code */
2021         diag = fm10k_init_shared_code(hw);
2022         if (diag != FM10K_SUCCESS) {
2023                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2024                 return -EIO;
2025         }
2026
2027         /*
2028          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2029          * there is no way to get link status without reading BAR4.  Until this
2030          * works, assume we have maximum bandwidth.
2031          * @todo - fix bus info
2032          */
2033         hw->bus_caps.speed = fm10k_bus_speed_8000;
2034         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2035         hw->bus_caps.payload = fm10k_bus_payload_512;
2036         hw->bus.speed = fm10k_bus_speed_8000;
2037         hw->bus.width = fm10k_bus_width_pcie_x8;
2038         hw->bus.payload = fm10k_bus_payload_256;
2039
2040         /* Initialize the hw */
2041         diag = fm10k_init_hw(hw);
2042         if (diag != FM10K_SUCCESS) {
2043                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2044                 return -EIO;
2045         }
2046
2047         /* Initialize MAC address(es) */
2048         dev->data->mac_addrs = rte_zmalloc("fm10k",
2049                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2050         if (dev->data->mac_addrs == NULL) {
2051                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2052                 return -ENOMEM;
2053         }
2054
2055         diag = fm10k_read_mac_addr(hw);
2056
2057         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2058                         &dev->data->mac_addrs[0]);
2059
2060         if (diag != FM10K_SUCCESS ||
2061                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2062
2063                 /* Generate a random addr */
2064                 eth_random_addr(hw->mac.addr);
2065                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2066                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2067                 &dev->data->mac_addrs[0]);
2068         }
2069
2070         /* Reset the hw statistics */
2071         fm10k_stats_reset(dev);
2072
2073         /* Reset the hw */
2074         diag = fm10k_reset_hw(hw);
2075         if (diag != FM10K_SUCCESS) {
2076                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2077                 return -EIO;
2078         }
2079
2080         /* Setup mailbox service */
2081         diag = fm10k_setup_mbx_service(hw);
2082         if (diag != FM10K_SUCCESS) {
2083                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2084                 return -EIO;
2085         }
2086
2087         /*PF/VF has different interrupt handling mechanism */
2088         if (hw->mac.type == fm10k_mac_pf) {
2089                 /* register callback func to eal lib */
2090                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2091                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2092
2093                 /* enable MISC interrupt */
2094                 fm10k_dev_enable_intr_pf(dev);
2095         } else { /* VF */
2096                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2097                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2098
2099                 fm10k_dev_enable_intr_vf(dev);
2100         }
2101
2102         /* Enable uio intr after callback registered */
2103         rte_intr_enable(&(dev->pci_dev->intr_handle));
2104
2105         hw->mac.ops.update_int_moderator(hw);
2106
2107         /* Make sure Switch Manager is ready before going forward. */
2108         if (hw->mac.type == fm10k_mac_pf) {
2109                 int switch_ready = 0;
2110                 int i;
2111
2112                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2113                         fm10k_mbx_lock(hw);
2114                         hw->mac.ops.get_host_state(hw, &switch_ready);
2115                         fm10k_mbx_unlock(hw);
2116                         if (switch_ready)
2117                                 break;
2118                         /* Delay some time to acquire async LPORT_MAP info. */
2119                         rte_delay_us(WAIT_SWITCH_MSG_US);
2120                 }
2121
2122                 if (switch_ready == 0) {
2123                         PMD_INIT_LOG(ERR, "switch is not ready");
2124                         return -1;
2125                 }
2126         }
2127
2128         /*
2129          * Below function will trigger operations on mailbox, acquire lock to
2130          * avoid race condition from interrupt handler. Operations on mailbox
2131          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2132          * will handle and generate an interrupt to our side. Then,  FIFO in
2133          * mailbox will be touched.
2134          */
2135         fm10k_mbx_lock(hw);
2136         /* Enable port first */
2137         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2138
2139         /* Set unicast mode by default. App can change to other mode in other
2140          * API func.
2141          */
2142         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2143                                         FM10K_XCAST_MODE_NONE);
2144
2145         fm10k_mbx_unlock(hw);
2146
2147
2148         return 0;
2149 }
2150
2151 /*
2152  * The set of PCI devices this driver supports. This driver will enable both PF
2153  * and SRIOV-VF devices.
2154  */
2155 static const struct rte_pci_id pci_id_fm10k_map[] = {
2156 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2157 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2158 #include "rte_pci_dev_ids.h"
2159         { .vendor_id = 0, /* sentinel */ },
2160 };
2161
2162 static struct eth_driver rte_pmd_fm10k = {
2163         .pci_drv = {
2164                 .name = "rte_pmd_fm10k",
2165                 .id_table = pci_id_fm10k_map,
2166                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
2167         },
2168         .eth_dev_init = eth_fm10k_dev_init,
2169         .dev_private_size = sizeof(struct fm10k_adapter),
2170 };
2171
2172 /*
2173  * Driver initialization routine.
2174  * Invoked once at EAL init time.
2175  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2176  */
2177 static int
2178 rte_pmd_fm10k_init(__rte_unused const char *name,
2179         __rte_unused const char *params)
2180 {
2181         PMD_INIT_FUNC_TRACE();
2182         rte_eth_driver_register(&rte_pmd_fm10k);
2183         return 0;
2184 }
2185
2186 static struct rte_driver rte_fm10k_driver = {
2187         .type = PMD_PDEV,
2188         .init = rte_pmd_fm10k_init,
2189 };
2190
2191 PMD_REGISTER_DRIVER(rte_fm10k_driver);