drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void
  65 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add);
  66 static void
  67 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         return 0;
 125 }
 126
 127 /*
 128  * clean queue, descriptor rings, free software buffers used when stopping
 129  * device.
 130  */
 131 static inline void
 132 rx_queue_clean(struct fm10k_rx_queue *q)
 133 {
 134         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 135         uint32_t i;
 136         PMD_INIT_FUNC_TRACE();
 137
 138         /* zero descriptor rings */
 139         for (i = 0; i < q->nb_desc; ++i)
 140                 q->hw_ring[i] = zero;
 141
 142         /* free software buffers */
 143         for (i = 0; i < q->nb_desc; ++i) {
 144                 if (q->sw_ring[i]) {
 145                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 146                         q->sw_ring[i] = NULL;
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * free all queue memory used when releasing the queue (i.e. configure)
 153  */
 154 static inline void
 155 rx_queue_free(struct fm10k_rx_queue *q)
 156 {
 157         PMD_INIT_FUNC_TRACE();
 158         if (q) {
 159                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 160                 rx_queue_clean(q);
 161                 if (q->sw_ring) {
 162                         rte_free(q->sw_ring);
 163                         q->sw_ring = NULL;
 164                 }
 165                 rte_free(q);
 166                 q = NULL;
 167         }
 168 }
 169
 170 /*
 171  * disable RX queue, wait unitl HW finished necessary flush operation
 172  */
 173 static inline int
 174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 175 {
 176         uint32_t reg, i;
 177
 178         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 179         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 180                         reg & ~FM10K_RXQCTL_ENABLE);
 181
 182         /* Wait 100us at most */
 183         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 184                 rte_delay_us(1);
 185                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 186                 if (!(reg & FM10K_RXQCTL_ENABLE))
 187                         break;
 188         }
 189
 190         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 191                 return -1;
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * reset queue to initial state, allocate software buffers used when starting
 198  * device
 199  */
 200 static inline void
 201 tx_queue_reset(struct fm10k_tx_queue *q)
 202 {
 203         PMD_INIT_FUNC_TRACE();
 204         q->last_free = 0;
 205         q->next_free = 0;
 206         q->nb_used = 0;
 207         q->nb_free = q->nb_desc - 1;
 208         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 209         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 210 }
 211
 212 /*
 213  * clean queue, descriptor rings, free software buffers used when stopping
 214  * device
 215  */
 216 static inline void
 217 tx_queue_clean(struct fm10k_tx_queue *q)
 218 {
 219         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 220         uint32_t i;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         /* zero descriptor rings */
 224         for (i = 0; i < q->nb_desc; ++i)
 225                 q->hw_ring[i] = zero;
 226
 227         /* free software buffers */
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 if (q->sw_ring[i]) {
 230                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 231                         q->sw_ring[i] = NULL;
 232                 }
 233         }
 234 }
 235
 236 /*
 237  * free all queue memory used when releasing the queue (i.e. configure)
 238  */
 239 static inline void
 240 tx_queue_free(struct fm10k_tx_queue *q)
 241 {
 242         PMD_INIT_FUNC_TRACE();
 243         if (q) {
 244                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 245                 tx_queue_clean(q);
 246                 if (q->rs_tracker.list) {
 247                         rte_free(q->rs_tracker.list);
 248                         q->rs_tracker.list = NULL;
 249                 }
 250                 if (q->sw_ring) {
 251                         rte_free(q->sw_ring);
 252                         q->sw_ring = NULL;
 253                 }
 254                 rte_free(q);
 255                 q = NULL;
 256         }
 257 }
 258
 259 /*
 260  * disable TX queue, wait unitl HW finished necessary flush operation
 261  */
 262 static inline int
 263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 264 {
 265         uint32_t reg, i;
 266
 267         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 268         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 269                         reg & ~FM10K_TXDCTL_ENABLE);
 270
 271         /* Wait 100us at most */
 272         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 273                 rte_delay_us(1);
 274                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 275                 if (!(reg & FM10K_TXDCTL_ENABLE))
 276                         break;
 277         }
 278
 279         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 280                 return -1;
 281
 282         return 0;
 283 }
 284
 285 static int
 286 fm10k_dev_configure(struct rte_eth_dev *dev)
 287 {
 288         PMD_INIT_FUNC_TRACE();
 289
 290         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 291                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 292
 293         return 0;
 294 }
 295
 296 static void
 297 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 298 {
 299         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 300         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 301         uint32_t mrqc, *key, i, reta, j;
 302         uint64_t hf;
 303
 304 #define RSS_KEY_SIZE 40
 305         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 306                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 307                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 308                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 309                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 310                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 311         };
 312
 313         if (dev->data->nb_rx_queues == 1 ||
 314             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 315             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 316                 return;
 317
 318         /* random key is rss_intel_key (default) or user provided (rss_key) */
 319         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 320                 key = (uint32_t *)rss_intel_key;
 321         else
 322                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 323
 324         /* Now fill our hash function seeds, 4 bytes at a time */
 325         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 326                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 327
 328         /*
 329          * Fill in redirection table
 330          * The byte-swap is needed because NIC registers are in
 331          * little-endian order.
 332          */
 333         reta = 0;
 334         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 335                 if (j == dev->data->nb_rx_queues)
 336                         j = 0;
 337                 reta = (reta << CHAR_BIT) | j;
 338                 if ((i & 3) == 3)
 339                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 340                                         rte_bswap32(reta));
 341         }
 342
 343         /*
 344          * Generate RSS hash based on packet types, TCP/UDP
 345          * port numbers and/or IPv4/v6 src and dst addresses
 346          */
 347         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 348         mrqc = 0;
 349         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 350         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 351         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 352         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 353         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 354         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 355         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 356         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 357         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 358
 359         if (mrqc == 0) {
 360                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 361                         "supported", hf);
 362                 return;
 363         }
 364
 365         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 366 }
 367
 368 static int
 369 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 370 {
 371         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 372         int i, ret;
 373         struct fm10k_tx_queue *txq;
 374         uint64_t base_addr;
 375         uint32_t size;
 376
 377         /* Disable TXINT to avoid possible interrupt */
 378         for (i = 0; i < hw->mac.max_queues; i++)
 379                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 380                                 3 << FM10K_TXINT_TIMER_SHIFT);
 381
 382         /* Setup TX queue */
 383         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 384                 txq = dev->data->tx_queues[i];
 385                 base_addr = txq->hw_ring_phys_addr;
 386                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 387
 388                 /* disable queue to avoid issues while updating state */
 389                 ret = tx_queue_disable(hw, i);
 390                 if (ret) {
 391                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 392                         return -1;
 393                 }
 394
 395                 /* set location and size for descriptor ring */
 396                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 397                                 base_addr & UINT64_LOWER_32BITS_MASK);
 398                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 399                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 400                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 401         }
 402         return 0;
 403 }
 404
 405 static int
 406 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 407 {
 408         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 409         int i, ret;
 410         struct fm10k_rx_queue *rxq;
 411         uint64_t base_addr;
 412         uint32_t size;
 413         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 414         uint16_t buf_size;
 415
 416         /* Disable RXINT to avoid possible interrupt */
 417         for (i = 0; i < hw->mac.max_queues; i++)
 418                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 419                                 3 << FM10K_RXINT_TIMER_SHIFT);
 420
 421         /* Setup RX queues */
 422         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 423                 rxq = dev->data->rx_queues[i];
 424                 base_addr = rxq->hw_ring_phys_addr;
 425                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 426
 427                 /* disable queue to avoid issues while updating state */
 428                 ret = rx_queue_disable(hw, i);
 429                 if (ret) {
 430                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 431                         return -1;
 432                 }
 433
 434                 /* Setup the Base and Length of the Rx Descriptor Ring */
 435                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 436                                 base_addr & UINT64_LOWER_32BITS_MASK);
 437                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 438                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 439                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 440
 441                 /* Configure the Rx buffer size for one buff without split */
 442                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 443                         RTE_PKTMBUF_HEADROOM);
 444                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 445                  * reserved for this purpose, and the worst case could be 511B.
 446                  * But SRR reg assumes all buffers have the same size. In order
 447                  * to fill the gap, we'll have to consider the worst case and
 448                  * assume 512B is reserved. If we don't do so, it's possible
 449                  * for HW to overwrite data to next mbuf.
 450                  */
 451                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 452
 453                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 454                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 455
 456                 /* It adds dual VLAN length for supporting dual VLAN */
 457                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 458                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 459                         dev->data->dev_conf.rxmode.enable_scatter) {
 460                         uint32_t reg;
 461                         dev->data->scattered_rx = 1;
 462                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 463                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 464                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 465                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 466                 }
 467
 468                 /* Enable drop on empty, it's RO for VF */
 469                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 470                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 471
 472                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 473                 FM10K_WRITE_FLUSH(hw);
 474         }
 475
 476         /* Configure RSS if applicable */
 477         fm10k_dev_mq_rx_configure(dev);
 478         return 0;
 479 }
 480
 481 static int
 482 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 483 {
 484         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 485         int err = -1;
 486         uint32_t reg;
 487         struct fm10k_rx_queue *rxq;
 488
 489         PMD_INIT_FUNC_TRACE();
 490
 491         if (rx_queue_id < dev->data->nb_rx_queues) {
 492                 rxq = dev->data->rx_queues[rx_queue_id];
 493                 err = rx_queue_reset(rxq);
 494                 if (err == -ENOMEM) {
 495                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 496                         return err;
 497                 } else if (err == -EINVAL) {
 498                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 499                                 " %d", err);
 500                         return err;
 501                 }
 502
 503                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 504                  * Note: this must be done AFTER the queue is enabled on real
 505                  * hardware, but BEFORE the queue is enabled when using the
 506                  * emulation platform. Do it in both places for now and remove
 507                  * this comment and the following two register writes when the
 508                  * emulation platform is no longer being used.
 509                  */
 510                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 511                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 512
 513                 /* Set PF ownership flag for PF devices */
 514                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 515                 if (hw->mac.type == fm10k_mac_pf)
 516                         reg |= FM10K_RXQCTL_PF;
 517                 reg |= FM10K_RXQCTL_ENABLE;
 518                 /* enable RX queue */
 519                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 520                 FM10K_WRITE_FLUSH(hw);
 521
 522                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 523                  * Note: this must be done AFTER the queue is enabled
 524                  */
 525                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 526                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 527         }
 528
 529         return err;
 530 }
 531
 532 static int
 533 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 534 {
 535         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 536
 537         PMD_INIT_FUNC_TRACE();
 538
 539         if (rx_queue_id < dev->data->nb_rx_queues) {
 540                 /* Disable RX queue */
 541                 rx_queue_disable(hw, rx_queue_id);
 542
 543                 /* Free mbuf and clean HW ring */
 544                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 545         }
 546
 547         return 0;
 548 }
 549
 550 static int
 551 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 552 {
 553         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 554         /** @todo - this should be defined in the shared code */
 555 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 556         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 557         int err = 0;
 558
 559         PMD_INIT_FUNC_TRACE();
 560
 561         if (tx_queue_id < dev->data->nb_tx_queues) {
 562                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 563
 564                 /* reset head and tail pointers */
 565                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 566                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 567
 568                 /* enable TX queue */
 569                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 570                                         FM10K_TXDCTL_ENABLE | txdctl);
 571                 FM10K_WRITE_FLUSH(hw);
 572         } else
 573                 err = -1;
 574
 575         return err;
 576 }
 577
 578 static int
 579 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 580 {
 581         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 582
 583         PMD_INIT_FUNC_TRACE();
 584
 585         if (tx_queue_id < dev->data->nb_tx_queues) {
 586                 tx_queue_disable(hw, tx_queue_id);
 587                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 588         }
 589
 590         return 0;
 591 }
 592
 593 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 594 {
 595         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 596                 != FM10K_DGLORTMAP_NONE);
 597 }
 598
 599 static void
 600 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 601 {
 602         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 603         int status;
 604
 605         PMD_INIT_FUNC_TRACE();
 606
 607         /* Return if it didn't acquire valid glort range */
 608         if (!fm10k_glort_valid(hw))
 609                 return;
 610
 611         fm10k_mbx_lock(hw);
 612         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 613                                 FM10K_XCAST_MODE_PROMISC);
 614         fm10k_mbx_unlock(hw);
 615
 616         if (status != FM10K_SUCCESS)
 617                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 618 }
 619
 620 static void
 621 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 622 {
 623         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 624         uint8_t mode;
 625         int status;
 626
 627         PMD_INIT_FUNC_TRACE();
 628
 629         /* Return if it didn't acquire valid glort range */
 630         if (!fm10k_glort_valid(hw))
 631                 return;
 632
 633         if (dev->data->all_multicast == 1)
 634                 mode = FM10K_XCAST_MODE_ALLMULTI;
 635         else
 636                 mode = FM10K_XCAST_MODE_NONE;
 637
 638         fm10k_mbx_lock(hw);
 639         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 640                                 mode);
 641         fm10k_mbx_unlock(hw);
 642
 643         if (status != FM10K_SUCCESS)
 644                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 645 }
 646
 647 static void
 648 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 649 {
 650         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 651         int status;
 652
 653         PMD_INIT_FUNC_TRACE();
 654
 655         /* Return if it didn't acquire valid glort range */
 656         if (!fm10k_glort_valid(hw))
 657                 return;
 658
 659         /* If promiscuous mode is enabled, it doesn't make sense to enable
 660          * allmulticast and disable promiscuous since fm10k only can select
 661          * one of the modes.
 662          */
 663         if (dev->data->promiscuous) {
 664                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 665                         "needn't enable allmulticast");
 666                 return;
 667         }
 668
 669         fm10k_mbx_lock(hw);
 670         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 671                                 FM10K_XCAST_MODE_ALLMULTI);
 672         fm10k_mbx_unlock(hw);
 673
 674         if (status != FM10K_SUCCESS)
 675                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 676 }
 677
 678 static void
 679 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 680 {
 681         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 682         int status;
 683
 684         PMD_INIT_FUNC_TRACE();
 685
 686         /* Return if it didn't acquire valid glort range */
 687         if (!fm10k_glort_valid(hw))
 688                 return;
 689
 690         if (dev->data->promiscuous) {
 691                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 692                         "since promisc mode is enabled");
 693                 return;
 694         }
 695
 696         fm10k_mbx_lock(hw);
 697         /* Change mode to unicast mode */
 698         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 699                                 FM10K_XCAST_MODE_NONE);
 700         fm10k_mbx_unlock(hw);
 701
 702         if (status != FM10K_SUCCESS)
 703                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 704 }
 705
 706 /* fls = find last set bit = 32 minus the number of leading zeros */
 707 #ifndef fls
 708 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 709 #endif
 710 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 711 static int
 712 fm10k_dev_start(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         int i, diag;
 716
 717         PMD_INIT_FUNC_TRACE();
 718
 719         /* stop, init, then start the hw */
 720         diag = fm10k_stop_hw(hw);
 721         if (diag != FM10K_SUCCESS) {
 722                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 723                 return -EIO;
 724         }
 725
 726         diag = fm10k_init_hw(hw);
 727         if (diag != FM10K_SUCCESS) {
 728                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 729                 return -EIO;
 730         }
 731
 732         diag = fm10k_start_hw(hw);
 733         if (diag != FM10K_SUCCESS) {
 734                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 735                 return -EIO;
 736         }
 737
 738         diag = fm10k_dev_tx_init(dev);
 739         if (diag) {
 740                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 741                 return diag;
 742         }
 743
 744         diag = fm10k_dev_rx_init(dev);
 745         if (diag) {
 746                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 747                 return diag;
 748         }
 749
 750         if (hw->mac.type == fm10k_mac_pf) {
 751                 /* Establish only VSI 0 as valid */
 752                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 753
 754                 /* Configure RSS bits used in RETA table */
 755                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 756                                 fls(dev->data->nb_rx_queues - 1) <<
 757                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 758
 759                 /* Invalidate all other GLORT entries */
 760                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 761                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 762                                         FM10K_DGLORTMAP_NONE);
 763         }
 764
 765         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 766                 struct fm10k_rx_queue *rxq;
 767                 rxq = dev->data->rx_queues[i];
 768
 769                 if (rxq->rx_deferred_start)
 770                         continue;
 771                 diag = fm10k_dev_rx_queue_start(dev, i);
 772                 if (diag != 0) {
 773                         int j;
 774                         for (j = 0; j < i; ++j)
 775                                 rx_queue_clean(dev->data->rx_queues[j]);
 776                         return diag;
 777                 }
 778         }
 779
 780         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 781                 struct fm10k_tx_queue *txq;
 782                 txq = dev->data->tx_queues[i];
 783
 784                 if (txq->tx_deferred_start)
 785                         continue;
 786                 diag = fm10k_dev_tx_queue_start(dev, i);
 787                 if (diag != 0) {
 788                         int j;
 789                         for (j = 0; j < i; ++j)
 790                                 tx_queue_clean(dev->data->tx_queues[j]);
 791                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 792                                 rx_queue_clean(dev->data->rx_queues[j]);
 793                         return diag;
 794                 }
 795         }
 796
 797         /* Update default vlan */
 798         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID)
 799                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 800
 801         return 0;
 802 }
 803
 804 static void
 805 fm10k_dev_stop(struct rte_eth_dev *dev)
 806 {
 807         int i;
 808
 809         PMD_INIT_FUNC_TRACE();
 810
 811         if (dev->data->tx_queues)
 812                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 813                         fm10k_dev_tx_queue_stop(dev, i);
 814
 815         if (dev->data->rx_queues)
 816                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 817                         fm10k_dev_rx_queue_stop(dev, i);
 818 }
 819
 820 static void
 821 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 822 {
 823         int i;
 824
 825         PMD_INIT_FUNC_TRACE();
 826
 827         if (dev->data->tx_queues) {
 828                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 829                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 830         }
 831
 832         if (dev->data->rx_queues) {
 833                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 834                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 835         }
 836 }
 837
 838 static void
 839 fm10k_dev_close(struct rte_eth_dev *dev)
 840 {
 841         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 842
 843         PMD_INIT_FUNC_TRACE();
 844
 845         fm10k_MACVLAN_remove_all(dev);
 846
 847         /* Stop mailbox service first */
 848         fm10k_close_mbx_service(hw);
 849         fm10k_dev_stop(dev);
 850         fm10k_dev_queue_release(dev);
 851         fm10k_stop_hw(hw);
 852 }
 853
 854 static int
 855 fm10k_link_update(struct rte_eth_dev *dev,
 856         __rte_unused int wait_to_complete)
 857 {
 858         PMD_INIT_FUNC_TRACE();
 859
 860         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 861          * x8 PCIe interface. For now, we leave the speed undefined since there
 862          * is no 50Gbps Ethernet. */
 863         dev->data->dev_link.link_speed  = 0;
 864         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 865         dev->data->dev_link.link_status = 1;
 866
 867         return 0;
 868 }
 869
 870 static void
 871 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 872 {
 873         uint64_t ipackets, opackets, ibytes, obytes;
 874         struct fm10k_hw *hw =
 875                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 876         struct fm10k_hw_stats *hw_stats =
 877                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 878         int i;
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         fm10k_update_hw_stats(hw, hw_stats);
 883
 884         ipackets = opackets = ibytes = obytes = 0;
 885         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 886                 (i < hw->mac.max_queues); ++i) {
 887                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 888                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 889                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 890                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 891                 ipackets += stats->q_ipackets[i];
 892                 opackets += stats->q_opackets[i];
 893                 ibytes   += stats->q_ibytes[i];
 894                 obytes   += stats->q_obytes[i];
 895         }
 896         stats->ipackets = ipackets;
 897         stats->opackets = opackets;
 898         stats->ibytes = ibytes;
 899         stats->obytes = obytes;
 900 }
 901
 902 static void
 903 fm10k_stats_reset(struct rte_eth_dev *dev)
 904 {
 905         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 906         struct fm10k_hw_stats *hw_stats =
 907                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 908
 909         PMD_INIT_FUNC_TRACE();
 910
 911         memset(hw_stats, 0, sizeof(*hw_stats));
 912         fm10k_rebind_hw_stats(hw, hw_stats);
 913 }
 914
 915 static void
 916 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 917         struct rte_eth_dev_info *dev_info)
 918 {
 919         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 920
 921         PMD_INIT_FUNC_TRACE();
 922
 923         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 924         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 925         dev_info->max_rx_queues      = hw->mac.max_queues;
 926         dev_info->max_tx_queues      = hw->mac.max_queues;
 927         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
 928         dev_info->max_hash_mac_addrs = 0;
 929         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 930         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 931         dev_info->rx_offload_capa =
 932                 DEV_RX_OFFLOAD_VLAN_STRIP |
 933                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 934                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 935                 DEV_RX_OFFLOAD_TCP_CKSUM;
 936         dev_info->tx_offload_capa =
 937                 DEV_TX_OFFLOAD_VLAN_INSERT |
 938                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
 939                 DEV_TX_OFFLOAD_UDP_CKSUM   |
 940                 DEV_TX_OFFLOAD_TCP_CKSUM   |
 941                 DEV_TX_OFFLOAD_TCP_TSO;
 942
 943         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
 944         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 945
 946         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 947                 .rx_thresh = {
 948                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 949                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 950                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 951                 },
 952                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 953                 .rx_drop_en = 0,
 954         };
 955
 956         dev_info->default_txconf = (struct rte_eth_txconf) {
 957                 .tx_thresh = {
 958                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 959                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 960                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 961                 },
 962                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 963                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 964                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 965                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 966         };
 967
 968 }
 969
 970 static int
 971 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 972 {
 973         s32 result;
 974         uint16_t mac_num = 0;
 975         uint32_t vid_idx, vid_bit, mac_index;
 976         struct fm10k_hw *hw;
 977         struct fm10k_macvlan_filter_info *macvlan;
 978         struct rte_eth_dev_data *data = dev->data;
 979
 980         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 981         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 982
 983         if (vlan_id > ETH_VLAN_ID_MAX) {
 984                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
 985                 return (-EINVAL);
 986         }
 987
 988         vid_idx = FM10K_VFTA_IDX(vlan_id);
 989         vid_bit = FM10K_VFTA_BIT(vlan_id);
 990         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
 991         if (on && (macvlan->vfta[vid_idx] & vid_bit))
 992                 return 0;
 993         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
 994         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
 995                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
 996                         "in the VLAN filter table");
 997                 return (-EINVAL);
 998         }
 999
1000         fm10k_mbx_lock(hw);
1001         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1002         fm10k_mbx_unlock(hw);
1003         if (result != FM10K_SUCCESS) {
1004                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1005                 return (-EIO);
1006         }
1007
1008         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1009                         (result == FM10K_SUCCESS); mac_index++) {
1010                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1011                         continue;
1012                 if (mac_num > macvlan->mac_num - 1) {
1013                         PMD_INIT_LOG(ERR, "MAC address number "
1014                                         "not match");
1015                         break;
1016                 }
1017                 fm10k_mbx_lock(hw);
1018                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1019                         data->mac_addrs[mac_index].addr_bytes,
1020                         vlan_id, on, 0);
1021                 fm10k_mbx_unlock(hw);
1022                 mac_num++;
1023         }
1024         if (result != FM10K_SUCCESS) {
1025                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1026                 return (-EIO);
1027         }
1028
1029         if (on) {
1030                 macvlan->vlan_num++;
1031                 macvlan->vfta[vid_idx] |= vid_bit;
1032         } else {
1033                 macvlan->vlan_num--;
1034                 macvlan->vfta[vid_idx] &= ~vid_bit;
1035         }
1036         return 0;
1037 }
1038
1039 static void
1040 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1041 {
1042         if (mask & ETH_VLAN_STRIP_MASK) {
1043                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1044                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1045                                         "always on in fm10k");
1046         }
1047
1048         if (mask & ETH_VLAN_EXTEND_MASK) {
1049                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1050                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1051                                         "supported in fm10k");
1052         }
1053
1054         if (mask & ETH_VLAN_FILTER_MASK) {
1055                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1056                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1057         }
1058 }
1059
1060 /* Add/Remove a MAC address, and update filters */
1061 static void
1062 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add)
1063 {
1064         uint32_t i, j, k;
1065         struct fm10k_hw *hw;
1066         struct fm10k_macvlan_filter_info *macvlan;
1067
1068         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1069         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1070
1071         i = 0;
1072         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1073                 if (macvlan->vfta[j]) {
1074                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1075                                 if (macvlan->vfta[j] & (1 << k)) {
1076                                         if (i + 1 > macvlan->vlan_num) {
1077                                                 PMD_INIT_LOG(ERR, "vlan number "
1078                                                                 "not match");
1079                                                 return;
1080                                         }
1081                                         fm10k_mbx_lock(hw);
1082                                         fm10k_update_uc_addr(hw,
1083                                                 hw->mac.dglort_map, mac,
1084                                                 j * FM10K_UINT32_BIT_SIZE + k,
1085                                                 add, 0);
1086                                         fm10k_mbx_unlock(hw);
1087                                         i++;
1088                                 }
1089                         }
1090                 }
1091         }
1092
1093         if (add)
1094                 macvlan->mac_num++;
1095         else
1096                 macvlan->mac_num--;
1097 }
1098
1099 /* Add a MAC address, and update filters */
1100 static void
1101 fm10k_macaddr_add(struct rte_eth_dev *dev,
1102                  struct ether_addr *mac_addr,
1103                  __rte_unused uint32_t index,
1104                  __rte_unused uint32_t pool)
1105 {
1106         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE);
1107 }
1108
1109 /* Remove a MAC address, and update filters */
1110 static void
1111 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1112 {
1113         struct rte_eth_dev_data *data = dev->data;
1114
1115         if (index < FM10K_MAX_MACADDR_NUM)
1116                 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1117                                 FALSE);
1118 }
1119
1120 /* Remove all VLAN and MAC address table entries */
1121 static void
1122 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev)
1123 {
1124         uint32_t j, k;
1125         struct fm10k_macvlan_filter_info *macvlan;
1126
1127         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1128         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1129                 if (macvlan->vfta[j]) {
1130                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1131                                 if (macvlan->vfta[j] & (1 << k))
1132                                         fm10k_vlan_filter_set(dev,
1133                                                 j * FM10K_UINT32_BIT_SIZE + k, false);
1134                         }
1135                 }
1136         }
1137 }
1138
1139 static inline int
1140 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1141 {
1142         if ((request < min) || (request > max) || ((request % mult) != 0))
1143                 return -1;
1144         else
1145                 return 0;
1146 }
1147
1148 /*
1149  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1150  * the physical address is required. If the memzone is already created, then
1151  * this function returns a pointer to the existing memzone.
1152  */
1153 static inline const struct rte_memzone *
1154 allocate_hw_ring(const char *driver_name, const char *ring_name,
1155         uint8_t port_id, uint16_t queue_id, int socket_id,
1156         uint32_t size, uint32_t align)
1157 {
1158         char name[RTE_MEMZONE_NAMESIZE];
1159         const struct rte_memzone *mz;
1160
1161         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1162                  driver_name, ring_name, port_id, queue_id, socket_id);
1163
1164         /* return the memzone if it already exists */
1165         mz = rte_memzone_lookup(name);
1166         if (mz)
1167                 return mz;
1168
1169 #ifdef RTE_LIBRTE_XEN_DOM0
1170         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1171                                            RTE_PGSIZE_2M);
1172 #else
1173         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1174 #endif
1175 }
1176
1177 static inline int
1178 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1179 {
1180         if ((request < min) || (request > max) || ((div % request) != 0))
1181                 return -1;
1182         else
1183                 return 0;
1184 }
1185
1186 static inline int
1187 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1188 {
1189         uint16_t rx_free_thresh;
1190
1191         if (conf->rx_free_thresh == 0)
1192                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1193         else
1194                 rx_free_thresh = conf->rx_free_thresh;
1195
1196         /* make sure the requested threshold satisfies the constraints */
1197         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1198                         FM10K_RX_FREE_THRESH_MAX(q),
1199                         FM10K_RX_FREE_THRESH_DIV(q),
1200                         rx_free_thresh)) {
1201                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1202                         "less than or equal to %u, "
1203                         "greater than or equal to %u, "
1204                         "and a divisor of %u",
1205                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1206                         FM10K_RX_FREE_THRESH_MIN(q),
1207                         FM10K_RX_FREE_THRESH_DIV(q));
1208                 return (-EINVAL);
1209         }
1210
1211         q->alloc_thresh = rx_free_thresh;
1212         q->drop_en = conf->rx_drop_en;
1213         q->rx_deferred_start = conf->rx_deferred_start;
1214
1215         return 0;
1216 }
1217
1218 /*
1219  * Hardware requires specific alignment for Rx packet buffers. At
1220  * least one of the following two conditions must be satisfied.
1221  *  1. Address is 512B aligned
1222  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1223  *
1224  * As such, the driver may need to adjust the DMA address within the
1225  * buffer by up to 512B.
1226  *
1227  * return 1 if the element size is valid, otherwise return 0.
1228  */
1229 static int
1230 mempool_element_size_valid(struct rte_mempool *mp)
1231 {
1232         uint32_t min_size;
1233
1234         /* elt_size includes mbuf header and headroom */
1235         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1236                         RTE_PKTMBUF_HEADROOM;
1237
1238         /* account for up to 512B of alignment */
1239         min_size -= FM10K_RX_DATABUF_ALIGN;
1240
1241         /* sanity check for overflow */
1242         if (min_size > mp->elt_size)
1243                 return 0;
1244
1245         /* size is valid */
1246         return 1;
1247 }
1248
1249 static int
1250 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1251         uint16_t nb_desc, unsigned int socket_id,
1252         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1253 {
1254         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1255         struct fm10k_rx_queue *q;
1256         const struct rte_memzone *mz;
1257
1258         PMD_INIT_FUNC_TRACE();
1259
1260         /* make sure the mempool element size can account for alignment. */
1261         if (!mempool_element_size_valid(mp)) {
1262                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1263                 return (-EINVAL);
1264         }
1265
1266         /* make sure a valid number of descriptors have been requested */
1267         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1268                                 FM10K_MULT_RX_DESC, nb_desc)) {
1269                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1270                         "less than or equal to %"PRIu32", "
1271                         "greater than or equal to %u, "
1272                         "and a multiple of %u",
1273                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1274                         FM10K_MULT_RX_DESC);
1275                 return (-EINVAL);
1276         }
1277
1278         /*
1279          * if this queue existed already, free the associated memory. The
1280          * queue cannot be reused in case we need to allocate memory on
1281          * different socket than was previously used.
1282          */
1283         if (dev->data->rx_queues[queue_id] != NULL) {
1284                 rx_queue_free(dev->data->rx_queues[queue_id]);
1285                 dev->data->rx_queues[queue_id] = NULL;
1286         }
1287
1288         /* allocate memory for the queue structure */
1289         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1290                                 socket_id);
1291         if (q == NULL) {
1292                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1293                 return (-ENOMEM);
1294         }
1295
1296         /* setup queue */
1297         q->mp = mp;
1298         q->nb_desc = nb_desc;
1299         q->port_id = dev->data->port_id;
1300         q->queue_id = queue_id;
1301         q->tail_ptr = (volatile uint32_t *)
1302                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1303         if (handle_rxconf(q, conf))
1304                 return (-EINVAL);
1305
1306         /* allocate memory for the software ring */
1307         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1308                                         nb_desc * sizeof(struct rte_mbuf *),
1309                                         RTE_CACHE_LINE_SIZE, socket_id);
1310         if (q->sw_ring == NULL) {
1311                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1312                 rte_free(q);
1313                 return (-ENOMEM);
1314         }
1315
1316         /*
1317          * allocate memory for the hardware descriptor ring. A memzone large
1318          * enough to hold the maximum ring size is requested to allow for
1319          * resizing in later calls to the queue setup function.
1320          */
1321         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1322                                 dev->data->port_id, queue_id, socket_id,
1323                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1324         if (mz == NULL) {
1325                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1326                 rte_free(q->sw_ring);
1327                 rte_free(q);
1328                 return (-ENOMEM);
1329         }
1330         q->hw_ring = mz->addr;
1331 #ifdef RTE_LIBRTE_XEN_DOM0
1332         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1333 #else
1334         q->hw_ring_phys_addr = mz->phys_addr;
1335 #endif
1336
1337         dev->data->rx_queues[queue_id] = q;
1338         return 0;
1339 }
1340
1341 static void
1342 fm10k_rx_queue_release(void *queue)
1343 {
1344         PMD_INIT_FUNC_TRACE();
1345
1346         rx_queue_free(queue);
1347 }
1348
1349 static inline int
1350 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1351 {
1352         uint16_t tx_free_thresh;
1353         uint16_t tx_rs_thresh;
1354
1355         /* constraint MACROs require that tx_free_thresh is configured
1356          * before tx_rs_thresh */
1357         if (conf->tx_free_thresh == 0)
1358                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1359         else
1360                 tx_free_thresh = conf->tx_free_thresh;
1361
1362         /* make sure the requested threshold satisfies the constraints */
1363         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1364                         FM10K_TX_FREE_THRESH_MAX(q),
1365                         FM10K_TX_FREE_THRESH_DIV(q),
1366                         tx_free_thresh)) {
1367                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1368                         "less than or equal to %u, "
1369                         "greater than or equal to %u, "
1370                         "and a divisor of %u",
1371                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1372                         FM10K_TX_FREE_THRESH_MIN(q),
1373                         FM10K_TX_FREE_THRESH_DIV(q));
1374                 return (-EINVAL);
1375         }
1376
1377         q->free_thresh = tx_free_thresh;
1378
1379         if (conf->tx_rs_thresh == 0)
1380                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1381         else
1382                 tx_rs_thresh = conf->tx_rs_thresh;
1383
1384         q->tx_deferred_start = conf->tx_deferred_start;
1385
1386         /* make sure the requested threshold satisfies the constraints */
1387         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1388                         FM10K_TX_RS_THRESH_MAX(q),
1389                         FM10K_TX_RS_THRESH_DIV(q),
1390                         tx_rs_thresh)) {
1391                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1392                         "less than or equal to %u, "
1393                         "greater than or equal to %u, "
1394                         "and a divisor of %u",
1395                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1396                         FM10K_TX_RS_THRESH_MIN(q),
1397                         FM10K_TX_RS_THRESH_DIV(q));
1398                 return (-EINVAL);
1399         }
1400
1401         q->rs_thresh = tx_rs_thresh;
1402
1403         return 0;
1404 }
1405
1406 static int
1407 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1408         uint16_t nb_desc, unsigned int socket_id,
1409         const struct rte_eth_txconf *conf)
1410 {
1411         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1412         struct fm10k_tx_queue *q;
1413         const struct rte_memzone *mz;
1414
1415         PMD_INIT_FUNC_TRACE();
1416
1417         /* make sure a valid number of descriptors have been requested */
1418         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1419                                 FM10K_MULT_TX_DESC, nb_desc)) {
1420                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1421                         "less than or equal to %"PRIu32", "
1422                         "greater than or equal to %u, "
1423                         "and a multiple of %u",
1424                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1425                         FM10K_MULT_TX_DESC);
1426                 return (-EINVAL);
1427         }
1428
1429         /*
1430          * if this queue existed already, free the associated memory. The
1431          * queue cannot be reused in case we need to allocate memory on
1432          * different socket than was previously used.
1433          */
1434         if (dev->data->tx_queues[queue_id] != NULL) {
1435                 tx_queue_free(dev->data->tx_queues[queue_id]);
1436                 dev->data->tx_queues[queue_id] = NULL;
1437         }
1438
1439         /* allocate memory for the queue structure */
1440         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1441                                 socket_id);
1442         if (q == NULL) {
1443                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1444                 return (-ENOMEM);
1445         }
1446
1447         /* setup queue */
1448         q->nb_desc = nb_desc;
1449         q->port_id = dev->data->port_id;
1450         q->queue_id = queue_id;
1451         q->tail_ptr = (volatile uint32_t *)
1452                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1453         if (handle_txconf(q, conf))
1454                 return (-EINVAL);
1455
1456         /* allocate memory for the software ring */
1457         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1458                                         nb_desc * sizeof(struct rte_mbuf *),
1459                                         RTE_CACHE_LINE_SIZE, socket_id);
1460         if (q->sw_ring == NULL) {
1461                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1462                 rte_free(q);
1463                 return (-ENOMEM);
1464         }
1465
1466         /*
1467          * allocate memory for the hardware descriptor ring. A memzone large
1468          * enough to hold the maximum ring size is requested to allow for
1469          * resizing in later calls to the queue setup function.
1470          */
1471         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1472                                 dev->data->port_id, queue_id, socket_id,
1473                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1474         if (mz == NULL) {
1475                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1476                 rte_free(q->sw_ring);
1477                 rte_free(q);
1478                 return (-ENOMEM);
1479         }
1480         q->hw_ring = mz->addr;
1481 #ifdef RTE_LIBRTE_XEN_DOM0
1482         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1483 #else
1484         q->hw_ring_phys_addr = mz->phys_addr;
1485 #endif
1486
1487         /*
1488          * allocate memory for the RS bit tracker. Enough slots to hold the
1489          * descriptor index for each RS bit needing to be set are required.
1490          */
1491         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1492                                 ((nb_desc + 1) / q->rs_thresh) *
1493                                 sizeof(uint16_t),
1494                                 RTE_CACHE_LINE_SIZE, socket_id);
1495         if (q->rs_tracker.list == NULL) {
1496                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1497                 rte_free(q->sw_ring);
1498                 rte_free(q);
1499                 return (-ENOMEM);
1500         }
1501
1502         dev->data->tx_queues[queue_id] = q;
1503         return 0;
1504 }
1505
1506 static void
1507 fm10k_tx_queue_release(void *queue)
1508 {
1509         PMD_INIT_FUNC_TRACE();
1510
1511         tx_queue_free(queue);
1512 }
1513
1514 static int
1515 fm10k_reta_update(struct rte_eth_dev *dev,
1516                         struct rte_eth_rss_reta_entry64 *reta_conf,
1517                         uint16_t reta_size)
1518 {
1519         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1520         uint16_t i, j, idx, shift;
1521         uint8_t mask;
1522         uint32_t reta;
1523
1524         PMD_INIT_FUNC_TRACE();
1525
1526         if (reta_size > FM10K_MAX_RSS_INDICES) {
1527                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1528                         "(%d) doesn't match the number hardware can supported "
1529                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1530                 return -EINVAL;
1531         }
1532
1533         /*
1534          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1535          * 128-entries in 32 registers
1536          */
1537         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1538                 idx = i / RTE_RETA_GROUP_SIZE;
1539                 shift = i % RTE_RETA_GROUP_SIZE;
1540                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1541                                 BIT_MASK_PER_UINT32);
1542                 if (mask == 0)
1543                         continue;
1544
1545                 reta = 0;
1546                 if (mask != BIT_MASK_PER_UINT32)
1547                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1548
1549                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1550                         if (mask & (0x1 << j)) {
1551                                 if (mask != 0xF)
1552                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1553                                 reta |= reta_conf[idx].reta[shift + j] <<
1554                                                 (CHAR_BIT * j);
1555                         }
1556                 }
1557                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1558         }
1559
1560         return 0;
1561 }
1562
1563 static int
1564 fm10k_reta_query(struct rte_eth_dev *dev,
1565                         struct rte_eth_rss_reta_entry64 *reta_conf,
1566                         uint16_t reta_size)
1567 {
1568         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1569         uint16_t i, j, idx, shift;
1570         uint8_t mask;
1571         uint32_t reta;
1572
1573         PMD_INIT_FUNC_TRACE();
1574
1575         if (reta_size < FM10K_MAX_RSS_INDICES) {
1576                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1577                         "(%d) doesn't match the number hardware can supported "
1578                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1579                 return -EINVAL;
1580         }
1581
1582         /*
1583          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1584          * 128-entries in 32 registers
1585          */
1586         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1587                 idx = i / RTE_RETA_GROUP_SIZE;
1588                 shift = i % RTE_RETA_GROUP_SIZE;
1589                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1590                                 BIT_MASK_PER_UINT32);
1591                 if (mask == 0)
1592                         continue;
1593
1594                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1595                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1596                         if (mask & (0x1 << j))
1597                                 reta_conf[idx].reta[shift + j] = ((reta >>
1598                                         CHAR_BIT * j) & UINT8_MAX);
1599                 }
1600         }
1601
1602         return 0;
1603 }
1604
1605 static int
1606 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1607         struct rte_eth_rss_conf *rss_conf)
1608 {
1609         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1610         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1611         uint32_t mrqc;
1612         uint64_t hf = rss_conf->rss_hf;
1613         int i;
1614
1615         PMD_INIT_FUNC_TRACE();
1616
1617         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1618                 FM10K_RSSRK_ENTRIES_PER_REG)
1619                 return -EINVAL;
1620
1621         if (hf == 0)
1622                 return -EINVAL;
1623
1624         mrqc = 0;
1625         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1626         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1627         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1628         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1629         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1630         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1631         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1632         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1633         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1634
1635         /* If the mapping doesn't fit any supported, return */
1636         if (mrqc == 0)
1637                 return -EINVAL;
1638
1639         if (key != NULL)
1640                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1641                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1642
1643         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1644
1645         return 0;
1646 }
1647
1648 static int
1649 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1650         struct rte_eth_rss_conf *rss_conf)
1651 {
1652         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1653         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1654         uint32_t mrqc;
1655         uint64_t hf;
1656         int i;
1657
1658         PMD_INIT_FUNC_TRACE();
1659
1660         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1661                                 FM10K_RSSRK_ENTRIES_PER_REG)
1662                 return -EINVAL;
1663
1664         if (key != NULL)
1665                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1666                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1667
1668         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1669         hf = 0;
1670         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1671         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1672         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1673         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1674         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1675         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1676         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1677         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1678         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1679
1680         rss_conf->rss_hf = hf;
1681
1682         return 0;
1683 }
1684
1685 static void
1686 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1687 {
1688         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1689         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1690
1691         /* Bind all local non-queue interrupt to vector 0 */
1692         int_map |= 0;
1693
1694         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1695         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1696         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1697         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1698         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1699         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1700
1701         /* Enable misc causes */
1702         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1703                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1704                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1705                                 FM10K_EIMR_ENABLE(MAILBOX) |
1706                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1707                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1708                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1709                                 FM10K_EIMR_ENABLE(VFLR));
1710
1711         /* Enable ITR 0 */
1712         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1713                                         FM10K_ITR_MASK_CLEAR);
1714         FM10K_WRITE_FLUSH(hw);
1715 }
1716
1717 static void
1718 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1719 {
1720         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1721         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1722
1723         int_map |= 0;
1724
1725         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1726         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1727         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1728         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1729         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1730         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1731
1732         /* Disable misc causes */
1733         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1734                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1735                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1736                                 FM10K_EIMR_DISABLE(MAILBOX) |
1737                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1738                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1739                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1740                                 FM10K_EIMR_DISABLE(VFLR));
1741
1742         /* Disable ITR 0 */
1743         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1744         FM10K_WRITE_FLUSH(hw);
1745 }
1746
1747 static void
1748 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1749 {
1750         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1751         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1752
1753         /* Bind all local non-queue interrupt to vector 0 */
1754         int_map |= 0;
1755
1756         /* Only INT 0 available, other 15 are reserved. */
1757         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1758
1759         /* Enable ITR 0 */
1760         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1761                                         FM10K_ITR_MASK_CLEAR);
1762         FM10K_WRITE_FLUSH(hw);
1763 }
1764
1765 static void
1766 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1767 {
1768         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1769         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1770
1771         int_map |= 0;
1772
1773         /* Only INT 0 available, other 15 are reserved. */
1774         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1775
1776         /* Disable ITR 0 */
1777         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1778         FM10K_WRITE_FLUSH(hw);
1779 }
1780
1781 static int
1782 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1783 {
1784         struct fm10k_fault fault;
1785         int err;
1786         const char *estr = "Unknown error";
1787
1788         /* Process PCA fault */
1789         if (eicr & FM10K_EICR_PCA_FAULT) {
1790                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1791                 if (err)
1792                         goto error;
1793                 switch (fault.type) {
1794                 case PCA_NO_FAULT:
1795                         estr = "PCA_NO_FAULT"; break;
1796                 case PCA_UNMAPPED_ADDR:
1797                         estr = "PCA_UNMAPPED_ADDR"; break;
1798                 case PCA_BAD_QACCESS_PF:
1799                         estr = "PCA_BAD_QACCESS_PF"; break;
1800                 case PCA_BAD_QACCESS_VF:
1801                         estr = "PCA_BAD_QACCESS_VF"; break;
1802                 case PCA_MALICIOUS_REQ:
1803                         estr = "PCA_MALICIOUS_REQ"; break;
1804                 case PCA_POISONED_TLP:
1805                         estr = "PCA_POISONED_TLP"; break;
1806                 case PCA_TLP_ABORT:
1807                         estr = "PCA_TLP_ABORT"; break;
1808                 default:
1809                         goto error;
1810                 }
1811                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1812                         estr, fault.func ? "VF" : "PF", fault.func,
1813                         fault.address, fault.specinfo);
1814         }
1815
1816         /* Process THI fault */
1817         if (eicr & FM10K_EICR_THI_FAULT) {
1818                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1819                 if (err)
1820                         goto error;
1821                 switch (fault.type) {
1822                 case THI_NO_FAULT:
1823                         estr = "THI_NO_FAULT"; break;
1824                 case THI_MAL_DIS_Q_FAULT:
1825                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1826                 default:
1827                         goto error;
1828                 }
1829                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1830                         estr, fault.func ? "VF" : "PF", fault.func,
1831                         fault.address, fault.specinfo);
1832         }
1833
1834         /* Process FUM fault */
1835         if (eicr & FM10K_EICR_FUM_FAULT) {
1836                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1837                 if (err)
1838                         goto error;
1839                 switch (fault.type) {
1840                 case FUM_NO_FAULT:
1841                         estr = "FUM_NO_FAULT"; break;
1842                 case FUM_UNMAPPED_ADDR:
1843                         estr = "FUM_UNMAPPED_ADDR"; break;
1844                 case FUM_POISONED_TLP:
1845                         estr = "FUM_POISONED_TLP"; break;
1846                 case FUM_BAD_VF_QACCESS:
1847                         estr = "FUM_BAD_VF_QACCESS"; break;
1848                 case FUM_ADD_DECODE_ERR:
1849                         estr = "FUM_ADD_DECODE_ERR"; break;
1850                 case FUM_RO_ERROR:
1851                         estr = "FUM_RO_ERROR"; break;
1852                 case FUM_QPRC_CRC_ERROR:
1853                         estr = "FUM_QPRC_CRC_ERROR"; break;
1854                 case FUM_CSR_TIMEOUT:
1855                         estr = "FUM_CSR_TIMEOUT"; break;
1856                 case FUM_INVALID_TYPE:
1857                         estr = "FUM_INVALID_TYPE"; break;
1858                 case FUM_INVALID_LENGTH:
1859                         estr = "FUM_INVALID_LENGTH"; break;
1860                 case FUM_INVALID_BE:
1861                         estr = "FUM_INVALID_BE"; break;
1862                 case FUM_INVALID_ALIGN:
1863                         estr = "FUM_INVALID_ALIGN"; break;
1864                 default:
1865                         goto error;
1866                 }
1867                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1868                         estr, fault.func ? "VF" : "PF", fault.func,
1869                         fault.address, fault.specinfo);
1870         }
1871
1872         return 0;
1873 error:
1874         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1875         return err;
1876 }
1877
1878 /**
1879  * PF interrupt handler triggered by NIC for handling specific interrupt.
1880  *
1881  * @param handle
1882  *  Pointer to interrupt handle.
1883  * @param param
1884  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1885  *
1886  * @return
1887  *  void
1888  */
1889 static void
1890 fm10k_dev_interrupt_handler_pf(
1891                         __rte_unused struct rte_intr_handle *handle,
1892                         void *param)
1893 {
1894         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1895         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1896         uint32_t cause, status;
1897
1898         if (hw->mac.type != fm10k_mac_pf)
1899                 return;
1900
1901         cause = FM10K_READ_REG(hw, FM10K_EICR);
1902
1903         /* Handle PCI fault cases */
1904         if (cause & FM10K_EICR_FAULT_MASK) {
1905                 PMD_INIT_LOG(ERR, "INT: find fault!");
1906                 fm10k_dev_handle_fault(hw, cause);
1907         }
1908
1909         /* Handle switch up/down */
1910         if (cause & FM10K_EICR_SWITCHNOTREADY)
1911                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1912
1913         if (cause & FM10K_EICR_SWITCHREADY)
1914                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1915
1916         /* Handle mailbox message */
1917         fm10k_mbx_lock(hw);
1918         hw->mbx.ops.process(hw, &hw->mbx);
1919         fm10k_mbx_unlock(hw);
1920
1921         /* Handle SRAM error */
1922         if (cause & FM10K_EICR_SRAMERROR) {
1923                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1924
1925                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1926                 /* Write to clear pending bits */
1927                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1928
1929                 /* Todo: print out error message after shared code  updates */
1930         }
1931
1932         /* Clear these 3 events if having any */
1933         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1934                  FM10K_EICR_SWITCHREADY;
1935         if (cause)
1936                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1937
1938         /* Re-enable interrupt from device side */
1939         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1940                                         FM10K_ITR_MASK_CLEAR);
1941         /* Re-enable interrupt from host side */
1942         rte_intr_enable(&(dev->pci_dev->intr_handle));
1943 }
1944
1945 /**
1946  * VF interrupt handler triggered by NIC for handling specific interrupt.
1947  *
1948  * @param handle
1949  *  Pointer to interrupt handle.
1950  * @param param
1951  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1952  *
1953  * @return
1954  *  void
1955  */
1956 static void
1957 fm10k_dev_interrupt_handler_vf(
1958                         __rte_unused struct rte_intr_handle *handle,
1959                         void *param)
1960 {
1961         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1962         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1963
1964         if (hw->mac.type != fm10k_mac_vf)
1965                 return;
1966
1967         /* Handle mailbox message if lock is acquired */
1968         fm10k_mbx_lock(hw);
1969         hw->mbx.ops.process(hw, &hw->mbx);
1970         fm10k_mbx_unlock(hw);
1971
1972         /* Re-enable interrupt from device side */
1973         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1974                                         FM10K_ITR_MASK_CLEAR);
1975         /* Re-enable interrupt from host side */
1976         rte_intr_enable(&(dev->pci_dev->intr_handle));
1977 }
1978
1979 /* Mailbox message handler in VF */
1980 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1981         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1982         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1983         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1984         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1985 };
1986
1987 /* Mailbox message handler in PF */
1988 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1989         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1990         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1991         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1992         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1993         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1994         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1995         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1996 };
1997
1998 static int
1999 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2000 {
2001         int err;
2002
2003         /* Initialize mailbox lock */
2004         fm10k_mbx_initlock(hw);
2005
2006         /* Replace default message handler with new ones */
2007         if (hw->mac.type == fm10k_mac_pf)
2008                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2009         else
2010                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2011
2012         if (err) {
2013                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2014                                 err);
2015                 return err;
2016         }
2017         /* Connect to SM for PF device or PF for VF device */
2018         return hw->mbx.ops.connect(hw, &hw->mbx);
2019 }
2020
2021 static void
2022 fm10k_close_mbx_service(struct fm10k_hw *hw)
2023 {
2024         /* Disconnect from SM for PF device or PF for VF device */
2025         hw->mbx.ops.disconnect(hw, &hw->mbx);
2026 }
2027
2028 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2029         .dev_configure          = fm10k_dev_configure,
2030         .dev_start              = fm10k_dev_start,
2031         .dev_stop               = fm10k_dev_stop,
2032         .dev_close              = fm10k_dev_close,
2033         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2034         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2035         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2036         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2037         .stats_get              = fm10k_stats_get,
2038         .stats_reset            = fm10k_stats_reset,
2039         .link_update            = fm10k_link_update,
2040         .dev_infos_get          = fm10k_dev_infos_get,
2041         .vlan_filter_set        = fm10k_vlan_filter_set,
2042         .vlan_offload_set       = fm10k_vlan_offload_set,
2043         .mac_addr_add           = fm10k_macaddr_add,
2044         .mac_addr_remove        = fm10k_macaddr_remove,
2045         .rx_queue_start         = fm10k_dev_rx_queue_start,
2046         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2047         .tx_queue_start         = fm10k_dev_tx_queue_start,
2048         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2049         .rx_queue_setup         = fm10k_rx_queue_setup,
2050         .rx_queue_release       = fm10k_rx_queue_release,
2051         .tx_queue_setup         = fm10k_tx_queue_setup,
2052         .tx_queue_release       = fm10k_tx_queue_release,
2053         .reta_update            = fm10k_reta_update,
2054         .reta_query             = fm10k_reta_query,
2055         .rss_hash_update        = fm10k_rss_hash_update,
2056         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2057 };
2058
2059 static int
2060 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2061 {
2062         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2063         int diag;
2064         struct fm10k_macvlan_filter_info *macvlan;
2065
2066         PMD_INIT_FUNC_TRACE();
2067
2068         dev->dev_ops = &fm10k_eth_dev_ops;
2069         dev->rx_pkt_burst = &fm10k_recv_pkts;
2070         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2071
2072         if (dev->data->scattered_rx)
2073                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2074
2075         /* only initialize in the primary process */
2076         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2077                 return 0;
2078
2079         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2080         memset(macvlan, 0, sizeof(*macvlan));
2081         /* Vendor and Device ID need to be set before init of shared code */
2082         memset(hw, 0, sizeof(*hw));
2083         hw->device_id = dev->pci_dev->id.device_id;
2084         hw->vendor_id = dev->pci_dev->id.vendor_id;
2085         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2086         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2087         hw->revision_id = 0;
2088         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2089         if (hw->hw_addr == NULL) {
2090                 PMD_INIT_LOG(ERR, "Bad mem resource."
2091                         " Try to blacklist unused devices.");
2092                 return -EIO;
2093         }
2094
2095         /* Store fm10k_adapter pointer */
2096         hw->back = dev->data->dev_private;
2097
2098         /* Initialize the shared code */
2099         diag = fm10k_init_shared_code(hw);
2100         if (diag != FM10K_SUCCESS) {
2101                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2102                 return -EIO;
2103         }
2104
2105         /*
2106          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2107          * there is no way to get link status without reading BAR4.  Until this
2108          * works, assume we have maximum bandwidth.
2109          * @todo - fix bus info
2110          */
2111         hw->bus_caps.speed = fm10k_bus_speed_8000;
2112         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2113         hw->bus_caps.payload = fm10k_bus_payload_512;
2114         hw->bus.speed = fm10k_bus_speed_8000;
2115         hw->bus.width = fm10k_bus_width_pcie_x8;
2116         hw->bus.payload = fm10k_bus_payload_256;
2117
2118         /* Initialize the hw */
2119         diag = fm10k_init_hw(hw);
2120         if (diag != FM10K_SUCCESS) {
2121                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2122                 return -EIO;
2123         }
2124
2125         /* Initialize MAC address(es) */
2126         dev->data->mac_addrs = rte_zmalloc("fm10k",
2127                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2128         if (dev->data->mac_addrs == NULL) {
2129                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2130                 return -ENOMEM;
2131         }
2132
2133         diag = fm10k_read_mac_addr(hw);
2134
2135         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2136                         &dev->data->mac_addrs[0]);
2137
2138         if (diag != FM10K_SUCCESS ||
2139                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2140
2141                 /* Generate a random addr */
2142                 eth_random_addr(hw->mac.addr);
2143                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2144                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2145                 &dev->data->mac_addrs[0]);
2146         }
2147
2148         /* Reset the hw statistics */
2149         fm10k_stats_reset(dev);
2150
2151         /* Reset the hw */
2152         diag = fm10k_reset_hw(hw);
2153         if (diag != FM10K_SUCCESS) {
2154                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2155                 return -EIO;
2156         }
2157
2158         /* Setup mailbox service */
2159         diag = fm10k_setup_mbx_service(hw);
2160         if (diag != FM10K_SUCCESS) {
2161                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2162                 return -EIO;
2163         }
2164
2165         /*PF/VF has different interrupt handling mechanism */
2166         if (hw->mac.type == fm10k_mac_pf) {
2167                 /* register callback func to eal lib */
2168                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2169                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2170
2171                 /* enable MISC interrupt */
2172                 fm10k_dev_enable_intr_pf(dev);
2173         } else { /* VF */
2174                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2175                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2176
2177                 fm10k_dev_enable_intr_vf(dev);
2178         }
2179
2180         /* Enable uio intr after callback registered */
2181         rte_intr_enable(&(dev->pci_dev->intr_handle));
2182
2183         hw->mac.ops.update_int_moderator(hw);
2184
2185         /* Make sure Switch Manager is ready before going forward. */
2186         if (hw->mac.type == fm10k_mac_pf) {
2187                 int switch_ready = 0;
2188                 int i;
2189
2190                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2191                         fm10k_mbx_lock(hw);
2192                         hw->mac.ops.get_host_state(hw, &switch_ready);
2193                         fm10k_mbx_unlock(hw);
2194                         if (switch_ready)
2195                                 break;
2196                         /* Delay some time to acquire async LPORT_MAP info. */
2197                         rte_delay_us(WAIT_SWITCH_MSG_US);
2198                 }
2199
2200                 if (switch_ready == 0) {
2201                         PMD_INIT_LOG(ERR, "switch is not ready");
2202                         return -1;
2203                 }
2204         }
2205
2206         /*
2207          * Below function will trigger operations on mailbox, acquire lock to
2208          * avoid race condition from interrupt handler. Operations on mailbox
2209          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2210          * will handle and generate an interrupt to our side. Then,  FIFO in
2211          * mailbox will be touched.
2212          */
2213         fm10k_mbx_lock(hw);
2214         /* Enable port first */
2215         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2216
2217         /* Set unicast mode by default. App can change to other mode in other
2218          * API func.
2219          */
2220         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2221                                         FM10K_XCAST_MODE_NONE);
2222
2223         fm10k_mbx_unlock(hw);
2224
2225         /* Add default mac address */
2226         fm10k_MAC_filter_set(dev, hw->mac.addr, true);
2227
2228         return 0;
2229 }
2230
2231 static int
2232 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2233 {
2234         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2235
2236         PMD_INIT_FUNC_TRACE();
2237
2238         /* only uninitialize in the primary process */
2239         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2240                 return 0;
2241
2242         /* safe to close dev here */
2243         fm10k_dev_close(dev);
2244
2245         dev->dev_ops = NULL;
2246         dev->rx_pkt_burst = NULL;
2247         dev->tx_pkt_burst = NULL;
2248
2249         /* disable uio/vfio intr */
2250         rte_intr_disable(&(dev->pci_dev->intr_handle));
2251
2252         /*PF/VF has different interrupt handling mechanism */
2253         if (hw->mac.type == fm10k_mac_pf) {
2254                 /* disable interrupt */
2255                 fm10k_dev_disable_intr_pf(dev);
2256
2257                 /* unregister callback func to eal lib */
2258                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2259                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2260         } else {
2261                 /* disable interrupt */
2262                 fm10k_dev_disable_intr_vf(dev);
2263
2264                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2265                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2266         }
2267
2268         /* free mac memory */
2269         if (dev->data->mac_addrs) {
2270                 rte_free(dev->data->mac_addrs);
2271                 dev->data->mac_addrs = NULL;
2272         }
2273
2274         memset(hw, 0, sizeof(*hw));
2275
2276         return 0;
2277 }
2278
2279 /*
2280  * The set of PCI devices this driver supports. This driver will enable both PF
2281  * and SRIOV-VF devices.
2282  */
2283 static const struct rte_pci_id pci_id_fm10k_map[] = {
2284 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2285 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2286 #include "rte_pci_dev_ids.h"
2287         { .vendor_id = 0, /* sentinel */ },
2288 };
2289
2290 static struct eth_driver rte_pmd_fm10k = {
2291         .pci_drv = {
2292                 .name = "rte_pmd_fm10k",
2293                 .id_table = pci_id_fm10k_map,
2294                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2295         },
2296         .eth_dev_init = eth_fm10k_dev_init,
2297         .eth_dev_uninit = eth_fm10k_dev_uninit,
2298         .dev_private_size = sizeof(struct fm10k_adapter),
2299 };
2300
2301 /*
2302  * Driver initialization routine.
2303  * Invoked once at EAL init time.
2304  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2305  */
2306 static int
2307 rte_pmd_fm10k_init(__rte_unused const char *name,
2308         __rte_unused const char *params)
2309 {
2310         PMD_INIT_FUNC_TRACE();
2311         rte_eth_driver_register(&rte_pmd_fm10k);
2312         return 0;
2313 }
2314
2315 static struct rte_driver rte_fm10k_driver = {
2316         .type = PMD_PDEV,
2317         .init = rte_pmd_fm10k_init,
2318 };
2319
2320 PMD_REGISTER_DRIVER(rte_fm10k_driver);