drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void
  65 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add);
  66 static void
  67 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         return 0;
 125 }
 126
 127 /*
 128  * clean queue, descriptor rings, free software buffers used when stopping
 129  * device.
 130  */
 131 static inline void
 132 rx_queue_clean(struct fm10k_rx_queue *q)
 133 {
 134         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 135         uint32_t i;
 136         PMD_INIT_FUNC_TRACE();
 137
 138         /* zero descriptor rings */
 139         for (i = 0; i < q->nb_desc; ++i)
 140                 q->hw_ring[i] = zero;
 141
 142         /* free software buffers */
 143         for (i = 0; i < q->nb_desc; ++i) {
 144                 if (q->sw_ring[i]) {
 145                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 146                         q->sw_ring[i] = NULL;
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * free all queue memory used when releasing the queue (i.e. configure)
 153  */
 154 static inline void
 155 rx_queue_free(struct fm10k_rx_queue *q)
 156 {
 157         PMD_INIT_FUNC_TRACE();
 158         if (q) {
 159                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 160                 rx_queue_clean(q);
 161                 if (q->sw_ring) {
 162                         rte_free(q->sw_ring);
 163                         q->sw_ring = NULL;
 164                 }
 165                 rte_free(q);
 166                 q = NULL;
 167         }
 168 }
 169
 170 /*
 171  * disable RX queue, wait unitl HW finished necessary flush operation
 172  */
 173 static inline int
 174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 175 {
 176         uint32_t reg, i;
 177
 178         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 179         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 180                         reg & ~FM10K_RXQCTL_ENABLE);
 181
 182         /* Wait 100us at most */
 183         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 184                 rte_delay_us(1);
 185                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 186                 if (!(reg & FM10K_RXQCTL_ENABLE))
 187                         break;
 188         }
 189
 190         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 191                 return -1;
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * reset queue to initial state, allocate software buffers used when starting
 198  * device
 199  */
 200 static inline void
 201 tx_queue_reset(struct fm10k_tx_queue *q)
 202 {
 203         PMD_INIT_FUNC_TRACE();
 204         q->last_free = 0;
 205         q->next_free = 0;
 206         q->nb_used = 0;
 207         q->nb_free = q->nb_desc - 1;
 208         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 209         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 210 }
 211
 212 /*
 213  * clean queue, descriptor rings, free software buffers used when stopping
 214  * device
 215  */
 216 static inline void
 217 tx_queue_clean(struct fm10k_tx_queue *q)
 218 {
 219         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 220         uint32_t i;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         /* zero descriptor rings */
 224         for (i = 0; i < q->nb_desc; ++i)
 225                 q->hw_ring[i] = zero;
 226
 227         /* free software buffers */
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 if (q->sw_ring[i]) {
 230                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 231                         q->sw_ring[i] = NULL;
 232                 }
 233         }
 234 }
 235
 236 /*
 237  * free all queue memory used when releasing the queue (i.e. configure)
 238  */
 239 static inline void
 240 tx_queue_free(struct fm10k_tx_queue *q)
 241 {
 242         PMD_INIT_FUNC_TRACE();
 243         if (q) {
 244                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 245                 tx_queue_clean(q);
 246                 if (q->rs_tracker.list) {
 247                         rte_free(q->rs_tracker.list);
 248                         q->rs_tracker.list = NULL;
 249                 }
 250                 if (q->sw_ring) {
 251                         rte_free(q->sw_ring);
 252                         q->sw_ring = NULL;
 253                 }
 254                 rte_free(q);
 255                 q = NULL;
 256         }
 257 }
 258
 259 /*
 260  * disable TX queue, wait unitl HW finished necessary flush operation
 261  */
 262 static inline int
 263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 264 {
 265         uint32_t reg, i;
 266
 267         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 268         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 269                         reg & ~FM10K_TXDCTL_ENABLE);
 270
 271         /* Wait 100us at most */
 272         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 273                 rte_delay_us(1);
 274                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 275                 if (!(reg & FM10K_TXDCTL_ENABLE))
 276                         break;
 277         }
 278
 279         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 280                 return -1;
 281
 282         return 0;
 283 }
 284
 285 static int
 286 fm10k_dev_configure(struct rte_eth_dev *dev)
 287 {
 288         PMD_INIT_FUNC_TRACE();
 289
 290         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 291                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 292
 293         return 0;
 294 }
 295
 296 static void
 297 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 298 {
 299         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 300         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 301         uint32_t mrqc, *key, i, reta, j;
 302         uint64_t hf;
 303
 304 #define RSS_KEY_SIZE 40
 305         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 306                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 307                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 308                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 309                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 310                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 311         };
 312
 313         if (dev->data->nb_rx_queues == 1 ||
 314             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 315             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 316                 return;
 317
 318         /* random key is rss_intel_key (default) or user provided (rss_key) */
 319         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 320                 key = (uint32_t *)rss_intel_key;
 321         else
 322                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 323
 324         /* Now fill our hash function seeds, 4 bytes at a time */
 325         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 326                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 327
 328         /*
 329          * Fill in redirection table
 330          * The byte-swap is needed because NIC registers are in
 331          * little-endian order.
 332          */
 333         reta = 0;
 334         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 335                 if (j == dev->data->nb_rx_queues)
 336                         j = 0;
 337                 reta = (reta << CHAR_BIT) | j;
 338                 if ((i & 3) == 3)
 339                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 340                                         rte_bswap32(reta));
 341         }
 342
 343         /*
 344          * Generate RSS hash based on packet types, TCP/UDP
 345          * port numbers and/or IPv4/v6 src and dst addresses
 346          */
 347         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 348         mrqc = 0;
 349         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 350         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 351         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 352         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 353         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 354         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 355         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 356         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 357         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 358
 359         if (mrqc == 0) {
 360                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 361                         "supported", hf);
 362                 return;
 363         }
 364
 365         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 366 }
 367
 368 static int
 369 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 370 {
 371         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 372         int i, ret;
 373         struct fm10k_tx_queue *txq;
 374         uint64_t base_addr;
 375         uint32_t size;
 376
 377         /* Disable TXINT to avoid possible interrupt */
 378         for (i = 0; i < hw->mac.max_queues; i++)
 379                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 380                                 3 << FM10K_TXINT_TIMER_SHIFT);
 381
 382         /* Setup TX queue */
 383         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 384                 txq = dev->data->tx_queues[i];
 385                 base_addr = txq->hw_ring_phys_addr;
 386                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 387
 388                 /* disable queue to avoid issues while updating state */
 389                 ret = tx_queue_disable(hw, i);
 390                 if (ret) {
 391                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 392                         return -1;
 393                 }
 394
 395                 /* set location and size for descriptor ring */
 396                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 397                                 base_addr & UINT64_LOWER_32BITS_MASK);
 398                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 399                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 400                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 401         }
 402         return 0;
 403 }
 404
 405 static int
 406 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 407 {
 408         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 409         int i, ret;
 410         struct fm10k_rx_queue *rxq;
 411         uint64_t base_addr;
 412         uint32_t size;
 413         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 414         uint16_t buf_size;
 415
 416         /* Disable RXINT to avoid possible interrupt */
 417         for (i = 0; i < hw->mac.max_queues; i++)
 418                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 419                                 3 << FM10K_RXINT_TIMER_SHIFT);
 420
 421         /* Setup RX queues */
 422         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 423                 rxq = dev->data->rx_queues[i];
 424                 base_addr = rxq->hw_ring_phys_addr;
 425                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 426
 427                 /* disable queue to avoid issues while updating state */
 428                 ret = rx_queue_disable(hw, i);
 429                 if (ret) {
 430                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 431                         return -1;
 432                 }
 433
 434                 /* Setup the Base and Length of the Rx Descriptor Ring */
 435                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 436                                 base_addr & UINT64_LOWER_32BITS_MASK);
 437                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 438                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 439                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 440
 441                 /* Configure the Rx buffer size for one buff without split */
 442                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 443                         RTE_PKTMBUF_HEADROOM);
 444                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 445                  * reserved for this purpose, and the worst case could be 511B.
 446                  * But SRR reg assumes all buffers have the same size. In order
 447                  * to fill the gap, we'll have to consider the worst case and
 448                  * assume 512B is reserved. If we don't do so, it's possible
 449                  * for HW to overwrite data to next mbuf.
 450                  */
 451                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 452
 453                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 454                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 455
 456                 /* It adds dual VLAN length for supporting dual VLAN */
 457                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 458                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 459                         dev->data->dev_conf.rxmode.enable_scatter) {
 460                         uint32_t reg;
 461                         dev->data->scattered_rx = 1;
 462                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 463                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 464                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 465                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 466                 }
 467
 468                 /* Enable drop on empty, it's RO for VF */
 469                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 470                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 471
 472                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 473                 FM10K_WRITE_FLUSH(hw);
 474         }
 475
 476         /* Configure RSS if applicable */
 477         fm10k_dev_mq_rx_configure(dev);
 478         return 0;
 479 }
 480
 481 static int
 482 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 483 {
 484         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 485         int err = -1;
 486         uint32_t reg;
 487         struct fm10k_rx_queue *rxq;
 488
 489         PMD_INIT_FUNC_TRACE();
 490
 491         if (rx_queue_id < dev->data->nb_rx_queues) {
 492                 rxq = dev->data->rx_queues[rx_queue_id];
 493                 err = rx_queue_reset(rxq);
 494                 if (err == -ENOMEM) {
 495                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 496                         return err;
 497                 } else if (err == -EINVAL) {
 498                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 499                                 " %d", err);
 500                         return err;
 501                 }
 502
 503                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 504                  * Note: this must be done AFTER the queue is enabled on real
 505                  * hardware, but BEFORE the queue is enabled when using the
 506                  * emulation platform. Do it in both places for now and remove
 507                  * this comment and the following two register writes when the
 508                  * emulation platform is no longer being used.
 509                  */
 510                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 511                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 512
 513                 /* Set PF ownership flag for PF devices */
 514                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 515                 if (hw->mac.type == fm10k_mac_pf)
 516                         reg |= FM10K_RXQCTL_PF;
 517                 reg |= FM10K_RXQCTL_ENABLE;
 518                 /* enable RX queue */
 519                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 520                 FM10K_WRITE_FLUSH(hw);
 521
 522                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 523                  * Note: this must be done AFTER the queue is enabled
 524                  */
 525                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 526                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 527         }
 528
 529         return err;
 530 }
 531
 532 static int
 533 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 534 {
 535         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 536
 537         PMD_INIT_FUNC_TRACE();
 538
 539         if (rx_queue_id < dev->data->nb_rx_queues) {
 540                 /* Disable RX queue */
 541                 rx_queue_disable(hw, rx_queue_id);
 542
 543                 /* Free mbuf and clean HW ring */
 544                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 545         }
 546
 547         return 0;
 548 }
 549
 550 static int
 551 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 552 {
 553         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 554         /** @todo - this should be defined in the shared code */
 555 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 556         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 557         int err = 0;
 558
 559         PMD_INIT_FUNC_TRACE();
 560
 561         if (tx_queue_id < dev->data->nb_tx_queues) {
 562                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 563
 564                 /* reset head and tail pointers */
 565                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 566                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 567
 568                 /* enable TX queue */
 569                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 570                                         FM10K_TXDCTL_ENABLE | txdctl);
 571                 FM10K_WRITE_FLUSH(hw);
 572         } else
 573                 err = -1;
 574
 575         return err;
 576 }
 577
 578 static int
 579 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 580 {
 581         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 582
 583         PMD_INIT_FUNC_TRACE();
 584
 585         if (tx_queue_id < dev->data->nb_tx_queues) {
 586                 tx_queue_disable(hw, tx_queue_id);
 587                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 588         }
 589
 590         return 0;
 591 }
 592
 593 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 594 {
 595         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 596                 != FM10K_DGLORTMAP_NONE);
 597 }
 598
 599 static void
 600 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 601 {
 602         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 603         int status;
 604
 605         PMD_INIT_FUNC_TRACE();
 606
 607         /* Return if it didn't acquire valid glort range */
 608         if (!fm10k_glort_valid(hw))
 609                 return;
 610
 611         fm10k_mbx_lock(hw);
 612         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 613                                 FM10K_XCAST_MODE_PROMISC);
 614         fm10k_mbx_unlock(hw);
 615
 616         if (status != FM10K_SUCCESS)
 617                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 618 }
 619
 620 static void
 621 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 622 {
 623         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 624         uint8_t mode;
 625         int status;
 626
 627         PMD_INIT_FUNC_TRACE();
 628
 629         /* Return if it didn't acquire valid glort range */
 630         if (!fm10k_glort_valid(hw))
 631                 return;
 632
 633         if (dev->data->all_multicast == 1)
 634                 mode = FM10K_XCAST_MODE_ALLMULTI;
 635         else
 636                 mode = FM10K_XCAST_MODE_NONE;
 637
 638         fm10k_mbx_lock(hw);
 639         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 640                                 mode);
 641         fm10k_mbx_unlock(hw);
 642
 643         if (status != FM10K_SUCCESS)
 644                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 645 }
 646
 647 static void
 648 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 649 {
 650         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 651         int status;
 652
 653         PMD_INIT_FUNC_TRACE();
 654
 655         /* Return if it didn't acquire valid glort range */
 656         if (!fm10k_glort_valid(hw))
 657                 return;
 658
 659         /* If promiscuous mode is enabled, it doesn't make sense to enable
 660          * allmulticast and disable promiscuous since fm10k only can select
 661          * one of the modes.
 662          */
 663         if (dev->data->promiscuous) {
 664                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 665                         "needn't enable allmulticast");
 666                 return;
 667         }
 668
 669         fm10k_mbx_lock(hw);
 670         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 671                                 FM10K_XCAST_MODE_ALLMULTI);
 672         fm10k_mbx_unlock(hw);
 673
 674         if (status != FM10K_SUCCESS)
 675                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 676 }
 677
 678 static void
 679 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 680 {
 681         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 682         int status;
 683
 684         PMD_INIT_FUNC_TRACE();
 685
 686         /* Return if it didn't acquire valid glort range */
 687         if (!fm10k_glort_valid(hw))
 688                 return;
 689
 690         if (dev->data->promiscuous) {
 691                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 692                         "since promisc mode is enabled");
 693                 return;
 694         }
 695
 696         fm10k_mbx_lock(hw);
 697         /* Change mode to unicast mode */
 698         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 699                                 FM10K_XCAST_MODE_NONE);
 700         fm10k_mbx_unlock(hw);
 701
 702         if (status != FM10K_SUCCESS)
 703                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 704 }
 705
 706 /* fls = find last set bit = 32 minus the number of leading zeros */
 707 #ifndef fls
 708 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 709 #endif
 710 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 711 static int
 712 fm10k_dev_start(struct rte_eth_dev *dev)
 713 {
 714         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 715         int i, diag;
 716
 717         PMD_INIT_FUNC_TRACE();
 718
 719         /* stop, init, then start the hw */
 720         diag = fm10k_stop_hw(hw);
 721         if (diag != FM10K_SUCCESS) {
 722                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 723                 return -EIO;
 724         }
 725
 726         diag = fm10k_init_hw(hw);
 727         if (diag != FM10K_SUCCESS) {
 728                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 729                 return -EIO;
 730         }
 731
 732         diag = fm10k_start_hw(hw);
 733         if (diag != FM10K_SUCCESS) {
 734                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 735                 return -EIO;
 736         }
 737
 738         diag = fm10k_dev_tx_init(dev);
 739         if (diag) {
 740                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 741                 return diag;
 742         }
 743
 744         diag = fm10k_dev_rx_init(dev);
 745         if (diag) {
 746                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 747                 return diag;
 748         }
 749
 750         if (hw->mac.type == fm10k_mac_pf) {
 751                 /* Establish only VSI 0 as valid */
 752                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 753
 754                 /* Configure RSS bits used in RETA table */
 755                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 756                                 fls(dev->data->nb_rx_queues - 1) <<
 757                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 758
 759                 /* Invalidate all other GLORT entries */
 760                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 761                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 762                                         FM10K_DGLORTMAP_NONE);
 763         }
 764
 765         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 766                 struct fm10k_rx_queue *rxq;
 767                 rxq = dev->data->rx_queues[i];
 768
 769                 if (rxq->rx_deferred_start)
 770                         continue;
 771                 diag = fm10k_dev_rx_queue_start(dev, i);
 772                 if (diag != 0) {
 773                         int j;
 774                         for (j = 0; j < i; ++j)
 775                                 rx_queue_clean(dev->data->rx_queues[j]);
 776                         return diag;
 777                 }
 778         }
 779
 780         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 781                 struct fm10k_tx_queue *txq;
 782                 txq = dev->data->tx_queues[i];
 783
 784                 if (txq->tx_deferred_start)
 785                         continue;
 786                 diag = fm10k_dev_tx_queue_start(dev, i);
 787                 if (diag != 0) {
 788                         int j;
 789                         for (j = 0; j < i; ++j)
 790                                 tx_queue_clean(dev->data->tx_queues[j]);
 791                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 792                                 rx_queue_clean(dev->data->rx_queues[j]);
 793                         return diag;
 794                 }
 795         }
 796
 797         /* Update default vlan */
 798         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID)
 799                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 800
 801         return 0;
 802 }
 803
 804 static void
 805 fm10k_dev_stop(struct rte_eth_dev *dev)
 806 {
 807         int i;
 808
 809         PMD_INIT_FUNC_TRACE();
 810
 811         if (dev->data->tx_queues)
 812                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 813                         fm10k_dev_tx_queue_stop(dev, i);
 814
 815         if (dev->data->rx_queues)
 816                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 817                         fm10k_dev_rx_queue_stop(dev, i);
 818 }
 819
 820 static void
 821 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 822 {
 823         int i;
 824
 825         PMD_INIT_FUNC_TRACE();
 826
 827         if (dev->data->tx_queues) {
 828                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 829                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 830         }
 831
 832         if (dev->data->rx_queues) {
 833                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 834                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 835         }
 836 }
 837
 838 static void
 839 fm10k_dev_close(struct rte_eth_dev *dev)
 840 {
 841         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 842
 843         PMD_INIT_FUNC_TRACE();
 844
 845         fm10k_MACVLAN_remove_all(dev);
 846
 847         /* Stop mailbox service first */
 848         fm10k_close_mbx_service(hw);
 849         fm10k_dev_stop(dev);
 850         fm10k_dev_queue_release(dev);
 851         fm10k_stop_hw(hw);
 852 }
 853
 854 static int
 855 fm10k_link_update(struct rte_eth_dev *dev,
 856         __rte_unused int wait_to_complete)
 857 {
 858         PMD_INIT_FUNC_TRACE();
 859
 860         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 861          * x8 PCIe interface. For now, we leave the speed undefined since there
 862          * is no 50Gbps Ethernet. */
 863         dev->data->dev_link.link_speed  = 0;
 864         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 865         dev->data->dev_link.link_status = 1;
 866
 867         return 0;
 868 }
 869
 870 static void
 871 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 872 {
 873         uint64_t ipackets, opackets, ibytes, obytes;
 874         struct fm10k_hw *hw =
 875                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 876         struct fm10k_hw_stats *hw_stats =
 877                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 878         int i;
 879
 880         PMD_INIT_FUNC_TRACE();
 881
 882         fm10k_update_hw_stats(hw, hw_stats);
 883
 884         ipackets = opackets = ibytes = obytes = 0;
 885         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 886                 (i < hw->mac.max_queues); ++i) {
 887                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 888                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 889                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 890                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 891                 ipackets += stats->q_ipackets[i];
 892                 opackets += stats->q_opackets[i];
 893                 ibytes   += stats->q_ibytes[i];
 894                 obytes   += stats->q_obytes[i];
 895         }
 896         stats->ipackets = ipackets;
 897         stats->opackets = opackets;
 898         stats->ibytes = ibytes;
 899         stats->obytes = obytes;
 900 }
 901
 902 static void
 903 fm10k_stats_reset(struct rte_eth_dev *dev)
 904 {
 905         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 906         struct fm10k_hw_stats *hw_stats =
 907                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 908
 909         PMD_INIT_FUNC_TRACE();
 910
 911         memset(hw_stats, 0, sizeof(*hw_stats));
 912         fm10k_rebind_hw_stats(hw, hw_stats);
 913 }
 914
 915 static void
 916 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 917         struct rte_eth_dev_info *dev_info)
 918 {
 919         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 920
 921         PMD_INIT_FUNC_TRACE();
 922
 923         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 924         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 925         dev_info->max_rx_queues      = hw->mac.max_queues;
 926         dev_info->max_tx_queues      = hw->mac.max_queues;
 927         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
 928         dev_info->max_hash_mac_addrs = 0;
 929         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 930         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 931         dev_info->rx_offload_capa =
 932                 DEV_RX_OFFLOAD_VLAN_STRIP |
 933                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 934                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 935                 DEV_RX_OFFLOAD_TCP_CKSUM;
 936         dev_info->tx_offload_capa =
 937                 DEV_TX_OFFLOAD_VLAN_INSERT |
 938                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
 939                 DEV_TX_OFFLOAD_UDP_CKSUM   |
 940                 DEV_TX_OFFLOAD_TCP_CKSUM;
 941
 942         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
 943         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 944
 945         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 946                 .rx_thresh = {
 947                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 948                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 949                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 950                 },
 951                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 952                 .rx_drop_en = 0,
 953         };
 954
 955         dev_info->default_txconf = (struct rte_eth_txconf) {
 956                 .tx_thresh = {
 957                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 958                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 959                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 960                 },
 961                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 962                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 963                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 964                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 965         };
 966
 967 }
 968
 969 static int
 970 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 971 {
 972         s32 result;
 973         uint16_t mac_num = 0;
 974         uint32_t vid_idx, vid_bit, mac_index;
 975         struct fm10k_hw *hw;
 976         struct fm10k_macvlan_filter_info *macvlan;
 977         struct rte_eth_dev_data *data = dev->data;
 978
 979         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 980         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 981
 982         if (vlan_id > ETH_VLAN_ID_MAX) {
 983                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
 984                 return (-EINVAL);
 985         }
 986
 987         vid_idx = FM10K_VFTA_IDX(vlan_id);
 988         vid_bit = FM10K_VFTA_BIT(vlan_id);
 989         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
 990         if (on && (macvlan->vfta[vid_idx] & vid_bit))
 991                 return 0;
 992         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
 993         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
 994                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
 995                         "in the VLAN filter table");
 996                 return (-EINVAL);
 997         }
 998
 999         fm10k_mbx_lock(hw);
1000         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1001         fm10k_mbx_unlock(hw);
1002         if (result != FM10K_SUCCESS) {
1003                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1004                 return (-EIO);
1005         }
1006
1007         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1008                         (result == FM10K_SUCCESS); mac_index++) {
1009                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1010                         continue;
1011                 if (mac_num > macvlan->mac_num - 1) {
1012                         PMD_INIT_LOG(ERR, "MAC address number "
1013                                         "not match");
1014                         break;
1015                 }
1016                 fm10k_mbx_lock(hw);
1017                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1018                         data->mac_addrs[mac_index].addr_bytes,
1019                         vlan_id, on, 0);
1020                 fm10k_mbx_unlock(hw);
1021                 mac_num++;
1022         }
1023         if (result != FM10K_SUCCESS) {
1024                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1025                 return (-EIO);
1026         }
1027
1028         if (on) {
1029                 macvlan->vlan_num++;
1030                 macvlan->vfta[vid_idx] |= vid_bit;
1031         } else {
1032                 macvlan->vlan_num--;
1033                 macvlan->vfta[vid_idx] &= ~vid_bit;
1034         }
1035         return 0;
1036 }
1037
1038 static void
1039 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1040 {
1041         if (mask & ETH_VLAN_STRIP_MASK) {
1042                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1043                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1044                                         "always on in fm10k");
1045         }
1046
1047         if (mask & ETH_VLAN_EXTEND_MASK) {
1048                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1049                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1050                                         "supported in fm10k");
1051         }
1052
1053         if (mask & ETH_VLAN_FILTER_MASK) {
1054                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1055                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1056         }
1057 }
1058
1059 /* Add/Remove a MAC address, and update filters */
1060 static void
1061 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add)
1062 {
1063         uint32_t i, j, k;
1064         struct fm10k_hw *hw;
1065         struct fm10k_macvlan_filter_info *macvlan;
1066
1067         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1068         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1069
1070         i = 0;
1071         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1072                 if (macvlan->vfta[j]) {
1073                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1074                                 if (macvlan->vfta[j] & (1 << k)) {
1075                                         if (i + 1 > macvlan->vlan_num) {
1076                                                 PMD_INIT_LOG(ERR, "vlan number "
1077                                                                 "not match");
1078                                                 return;
1079                                         }
1080                                         fm10k_mbx_lock(hw);
1081                                         fm10k_update_uc_addr(hw,
1082                                                 hw->mac.dglort_map, mac,
1083                                                 j * FM10K_UINT32_BIT_SIZE + k,
1084                                                 add, 0);
1085                                         fm10k_mbx_unlock(hw);
1086                                         i++;
1087                                 }
1088                         }
1089                 }
1090         }
1091
1092         if (add)
1093                 macvlan->mac_num++;
1094         else
1095                 macvlan->mac_num--;
1096 }
1097
1098 /* Add a MAC address, and update filters */
1099 static void
1100 fm10k_macaddr_add(struct rte_eth_dev *dev,
1101                  struct ether_addr *mac_addr,
1102                  __rte_unused uint32_t index,
1103                  __rte_unused uint32_t pool)
1104 {
1105         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE);
1106 }
1107
1108 /* Remove a MAC address, and update filters */
1109 static void
1110 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1111 {
1112         struct rte_eth_dev_data *data = dev->data;
1113
1114         if (index < FM10K_MAX_MACADDR_NUM)
1115                 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1116                                 FALSE);
1117 }
1118
1119 /* Remove all VLAN and MAC address table entries */
1120 static void
1121 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev)
1122 {
1123         uint32_t j, k;
1124         struct fm10k_macvlan_filter_info *macvlan;
1125
1126         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1127         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1128                 if (macvlan->vfta[j]) {
1129                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1130                                 if (macvlan->vfta[j] & (1 << k))
1131                                         fm10k_vlan_filter_set(dev,
1132                                                 j * FM10K_UINT32_BIT_SIZE + k, false);
1133                         }
1134                 }
1135         }
1136 }
1137
1138 static inline int
1139 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1140 {
1141         if ((request < min) || (request > max) || ((request % mult) != 0))
1142                 return -1;
1143         else
1144                 return 0;
1145 }
1146
1147 /*
1148  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1149  * the physical address is required. If the memzone is already created, then
1150  * this function returns a pointer to the existing memzone.
1151  */
1152 static inline const struct rte_memzone *
1153 allocate_hw_ring(const char *driver_name, const char *ring_name,
1154         uint8_t port_id, uint16_t queue_id, int socket_id,
1155         uint32_t size, uint32_t align)
1156 {
1157         char name[RTE_MEMZONE_NAMESIZE];
1158         const struct rte_memzone *mz;
1159
1160         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1161                  driver_name, ring_name, port_id, queue_id, socket_id);
1162
1163         /* return the memzone if it already exists */
1164         mz = rte_memzone_lookup(name);
1165         if (mz)
1166                 return mz;
1167
1168 #ifdef RTE_LIBRTE_XEN_DOM0
1169         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1170                                            RTE_PGSIZE_2M);
1171 #else
1172         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1173 #endif
1174 }
1175
1176 static inline int
1177 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1178 {
1179         if ((request < min) || (request > max) || ((div % request) != 0))
1180                 return -1;
1181         else
1182                 return 0;
1183 }
1184
1185 static inline int
1186 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1187 {
1188         uint16_t rx_free_thresh;
1189
1190         if (conf->rx_free_thresh == 0)
1191                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1192         else
1193                 rx_free_thresh = conf->rx_free_thresh;
1194
1195         /* make sure the requested threshold satisfies the constraints */
1196         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1197                         FM10K_RX_FREE_THRESH_MAX(q),
1198                         FM10K_RX_FREE_THRESH_DIV(q),
1199                         rx_free_thresh)) {
1200                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1201                         "less than or equal to %u, "
1202                         "greater than or equal to %u, "
1203                         "and a divisor of %u",
1204                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1205                         FM10K_RX_FREE_THRESH_MIN(q),
1206                         FM10K_RX_FREE_THRESH_DIV(q));
1207                 return (-EINVAL);
1208         }
1209
1210         q->alloc_thresh = rx_free_thresh;
1211         q->drop_en = conf->rx_drop_en;
1212         q->rx_deferred_start = conf->rx_deferred_start;
1213
1214         return 0;
1215 }
1216
1217 /*
1218  * Hardware requires specific alignment for Rx packet buffers. At
1219  * least one of the following two conditions must be satisfied.
1220  *  1. Address is 512B aligned
1221  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1222  *
1223  * As such, the driver may need to adjust the DMA address within the
1224  * buffer by up to 512B.
1225  *
1226  * return 1 if the element size is valid, otherwise return 0.
1227  */
1228 static int
1229 mempool_element_size_valid(struct rte_mempool *mp)
1230 {
1231         uint32_t min_size;
1232
1233         /* elt_size includes mbuf header and headroom */
1234         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1235                         RTE_PKTMBUF_HEADROOM;
1236
1237         /* account for up to 512B of alignment */
1238         min_size -= FM10K_RX_DATABUF_ALIGN;
1239
1240         /* sanity check for overflow */
1241         if (min_size > mp->elt_size)
1242                 return 0;
1243
1244         /* size is valid */
1245         return 1;
1246 }
1247
1248 static int
1249 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1250         uint16_t nb_desc, unsigned int socket_id,
1251         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1252 {
1253         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1254         struct fm10k_rx_queue *q;
1255         const struct rte_memzone *mz;
1256
1257         PMD_INIT_FUNC_TRACE();
1258
1259         /* make sure the mempool element size can account for alignment. */
1260         if (!mempool_element_size_valid(mp)) {
1261                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1262                 return (-EINVAL);
1263         }
1264
1265         /* make sure a valid number of descriptors have been requested */
1266         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1267                                 FM10K_MULT_RX_DESC, nb_desc)) {
1268                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1269                         "less than or equal to %"PRIu32", "
1270                         "greater than or equal to %u, "
1271                         "and a multiple of %u",
1272                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1273                         FM10K_MULT_RX_DESC);
1274                 return (-EINVAL);
1275         }
1276
1277         /*
1278          * if this queue existed already, free the associated memory. The
1279          * queue cannot be reused in case we need to allocate memory on
1280          * different socket than was previously used.
1281          */
1282         if (dev->data->rx_queues[queue_id] != NULL) {
1283                 rx_queue_free(dev->data->rx_queues[queue_id]);
1284                 dev->data->rx_queues[queue_id] = NULL;
1285         }
1286
1287         /* allocate memory for the queue structure */
1288         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1289                                 socket_id);
1290         if (q == NULL) {
1291                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1292                 return (-ENOMEM);
1293         }
1294
1295         /* setup queue */
1296         q->mp = mp;
1297         q->nb_desc = nb_desc;
1298         q->port_id = dev->data->port_id;
1299         q->queue_id = queue_id;
1300         q->tail_ptr = (volatile uint32_t *)
1301                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1302         if (handle_rxconf(q, conf))
1303                 return (-EINVAL);
1304
1305         /* allocate memory for the software ring */
1306         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1307                                         nb_desc * sizeof(struct rte_mbuf *),
1308                                         RTE_CACHE_LINE_SIZE, socket_id);
1309         if (q->sw_ring == NULL) {
1310                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1311                 rte_free(q);
1312                 return (-ENOMEM);
1313         }
1314
1315         /*
1316          * allocate memory for the hardware descriptor ring. A memzone large
1317          * enough to hold the maximum ring size is requested to allow for
1318          * resizing in later calls to the queue setup function.
1319          */
1320         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1321                                 dev->data->port_id, queue_id, socket_id,
1322                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1323         if (mz == NULL) {
1324                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1325                 rte_free(q->sw_ring);
1326                 rte_free(q);
1327                 return (-ENOMEM);
1328         }
1329         q->hw_ring = mz->addr;
1330 #ifdef RTE_LIBRTE_XEN_DOM0
1331         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1332 #else
1333         q->hw_ring_phys_addr = mz->phys_addr;
1334 #endif
1335
1336         dev->data->rx_queues[queue_id] = q;
1337         return 0;
1338 }
1339
1340 static void
1341 fm10k_rx_queue_release(void *queue)
1342 {
1343         PMD_INIT_FUNC_TRACE();
1344
1345         rx_queue_free(queue);
1346 }
1347
1348 static inline int
1349 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1350 {
1351         uint16_t tx_free_thresh;
1352         uint16_t tx_rs_thresh;
1353
1354         /* constraint MACROs require that tx_free_thresh is configured
1355          * before tx_rs_thresh */
1356         if (conf->tx_free_thresh == 0)
1357                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1358         else
1359                 tx_free_thresh = conf->tx_free_thresh;
1360
1361         /* make sure the requested threshold satisfies the constraints */
1362         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1363                         FM10K_TX_FREE_THRESH_MAX(q),
1364                         FM10K_TX_FREE_THRESH_DIV(q),
1365                         tx_free_thresh)) {
1366                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1367                         "less than or equal to %u, "
1368                         "greater than or equal to %u, "
1369                         "and a divisor of %u",
1370                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1371                         FM10K_TX_FREE_THRESH_MIN(q),
1372                         FM10K_TX_FREE_THRESH_DIV(q));
1373                 return (-EINVAL);
1374         }
1375
1376         q->free_thresh = tx_free_thresh;
1377
1378         if (conf->tx_rs_thresh == 0)
1379                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1380         else
1381                 tx_rs_thresh = conf->tx_rs_thresh;
1382
1383         q->tx_deferred_start = conf->tx_deferred_start;
1384
1385         /* make sure the requested threshold satisfies the constraints */
1386         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1387                         FM10K_TX_RS_THRESH_MAX(q),
1388                         FM10K_TX_RS_THRESH_DIV(q),
1389                         tx_rs_thresh)) {
1390                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1391                         "less than or equal to %u, "
1392                         "greater than or equal to %u, "
1393                         "and a divisor of %u",
1394                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1395                         FM10K_TX_RS_THRESH_MIN(q),
1396                         FM10K_TX_RS_THRESH_DIV(q));
1397                 return (-EINVAL);
1398         }
1399
1400         q->rs_thresh = tx_rs_thresh;
1401
1402         return 0;
1403 }
1404
1405 static int
1406 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1407         uint16_t nb_desc, unsigned int socket_id,
1408         const struct rte_eth_txconf *conf)
1409 {
1410         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1411         struct fm10k_tx_queue *q;
1412         const struct rte_memzone *mz;
1413
1414         PMD_INIT_FUNC_TRACE();
1415
1416         /* make sure a valid number of descriptors have been requested */
1417         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1418                                 FM10K_MULT_TX_DESC, nb_desc)) {
1419                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1420                         "less than or equal to %"PRIu32", "
1421                         "greater than or equal to %u, "
1422                         "and a multiple of %u",
1423                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1424                         FM10K_MULT_TX_DESC);
1425                 return (-EINVAL);
1426         }
1427
1428         /*
1429          * if this queue existed already, free the associated memory. The
1430          * queue cannot be reused in case we need to allocate memory on
1431          * different socket than was previously used.
1432          */
1433         if (dev->data->tx_queues[queue_id] != NULL) {
1434                 tx_queue_free(dev->data->tx_queues[queue_id]);
1435                 dev->data->tx_queues[queue_id] = NULL;
1436         }
1437
1438         /* allocate memory for the queue structure */
1439         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1440                                 socket_id);
1441         if (q == NULL) {
1442                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1443                 return (-ENOMEM);
1444         }
1445
1446         /* setup queue */
1447         q->nb_desc = nb_desc;
1448         q->port_id = dev->data->port_id;
1449         q->queue_id = queue_id;
1450         q->tail_ptr = (volatile uint32_t *)
1451                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1452         if (handle_txconf(q, conf))
1453                 return (-EINVAL);
1454
1455         /* allocate memory for the software ring */
1456         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1457                                         nb_desc * sizeof(struct rte_mbuf *),
1458                                         RTE_CACHE_LINE_SIZE, socket_id);
1459         if (q->sw_ring == NULL) {
1460                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1461                 rte_free(q);
1462                 return (-ENOMEM);
1463         }
1464
1465         /*
1466          * allocate memory for the hardware descriptor ring. A memzone large
1467          * enough to hold the maximum ring size is requested to allow for
1468          * resizing in later calls to the queue setup function.
1469          */
1470         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1471                                 dev->data->port_id, queue_id, socket_id,
1472                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1473         if (mz == NULL) {
1474                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1475                 rte_free(q->sw_ring);
1476                 rte_free(q);
1477                 return (-ENOMEM);
1478         }
1479         q->hw_ring = mz->addr;
1480 #ifdef RTE_LIBRTE_XEN_DOM0
1481         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1482 #else
1483         q->hw_ring_phys_addr = mz->phys_addr;
1484 #endif
1485
1486         /*
1487          * allocate memory for the RS bit tracker. Enough slots to hold the
1488          * descriptor index for each RS bit needing to be set are required.
1489          */
1490         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1491                                 ((nb_desc + 1) / q->rs_thresh) *
1492                                 sizeof(uint16_t),
1493                                 RTE_CACHE_LINE_SIZE, socket_id);
1494         if (q->rs_tracker.list == NULL) {
1495                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1496                 rte_free(q->sw_ring);
1497                 rte_free(q);
1498                 return (-ENOMEM);
1499         }
1500
1501         dev->data->tx_queues[queue_id] = q;
1502         return 0;
1503 }
1504
1505 static void
1506 fm10k_tx_queue_release(void *queue)
1507 {
1508         PMD_INIT_FUNC_TRACE();
1509
1510         tx_queue_free(queue);
1511 }
1512
1513 static int
1514 fm10k_reta_update(struct rte_eth_dev *dev,
1515                         struct rte_eth_rss_reta_entry64 *reta_conf,
1516                         uint16_t reta_size)
1517 {
1518         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1519         uint16_t i, j, idx, shift;
1520         uint8_t mask;
1521         uint32_t reta;
1522
1523         PMD_INIT_FUNC_TRACE();
1524
1525         if (reta_size > FM10K_MAX_RSS_INDICES) {
1526                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1527                         "(%d) doesn't match the number hardware can supported "
1528                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1529                 return -EINVAL;
1530         }
1531
1532         /*
1533          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1534          * 128-entries in 32 registers
1535          */
1536         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1537                 idx = i / RTE_RETA_GROUP_SIZE;
1538                 shift = i % RTE_RETA_GROUP_SIZE;
1539                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1540                                 BIT_MASK_PER_UINT32);
1541                 if (mask == 0)
1542                         continue;
1543
1544                 reta = 0;
1545                 if (mask != BIT_MASK_PER_UINT32)
1546                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1547
1548                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1549                         if (mask & (0x1 << j)) {
1550                                 if (mask != 0xF)
1551                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1552                                 reta |= reta_conf[idx].reta[shift + j] <<
1553                                                 (CHAR_BIT * j);
1554                         }
1555                 }
1556                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1557         }
1558
1559         return 0;
1560 }
1561
1562 static int
1563 fm10k_reta_query(struct rte_eth_dev *dev,
1564                         struct rte_eth_rss_reta_entry64 *reta_conf,
1565                         uint16_t reta_size)
1566 {
1567         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1568         uint16_t i, j, idx, shift;
1569         uint8_t mask;
1570         uint32_t reta;
1571
1572         PMD_INIT_FUNC_TRACE();
1573
1574         if (reta_size < FM10K_MAX_RSS_INDICES) {
1575                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1576                         "(%d) doesn't match the number hardware can supported "
1577                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1578                 return -EINVAL;
1579         }
1580
1581         /*
1582          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1583          * 128-entries in 32 registers
1584          */
1585         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1586                 idx = i / RTE_RETA_GROUP_SIZE;
1587                 shift = i % RTE_RETA_GROUP_SIZE;
1588                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1589                                 BIT_MASK_PER_UINT32);
1590                 if (mask == 0)
1591                         continue;
1592
1593                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1594                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1595                         if (mask & (0x1 << j))
1596                                 reta_conf[idx].reta[shift + j] = ((reta >>
1597                                         CHAR_BIT * j) & UINT8_MAX);
1598                 }
1599         }
1600
1601         return 0;
1602 }
1603
1604 static int
1605 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1606         struct rte_eth_rss_conf *rss_conf)
1607 {
1608         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1609         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1610         uint32_t mrqc;
1611         uint64_t hf = rss_conf->rss_hf;
1612         int i;
1613
1614         PMD_INIT_FUNC_TRACE();
1615
1616         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1617                 FM10K_RSSRK_ENTRIES_PER_REG)
1618                 return -EINVAL;
1619
1620         if (hf == 0)
1621                 return -EINVAL;
1622
1623         mrqc = 0;
1624         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1625         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1626         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1627         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1628         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1629         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1630         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1631         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1632         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1633
1634         /* If the mapping doesn't fit any supported, return */
1635         if (mrqc == 0)
1636                 return -EINVAL;
1637
1638         if (key != NULL)
1639                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1640                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1641
1642         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1643
1644         return 0;
1645 }
1646
1647 static int
1648 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1649         struct rte_eth_rss_conf *rss_conf)
1650 {
1651         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1652         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1653         uint32_t mrqc;
1654         uint64_t hf;
1655         int i;
1656
1657         PMD_INIT_FUNC_TRACE();
1658
1659         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1660                                 FM10K_RSSRK_ENTRIES_PER_REG)
1661                 return -EINVAL;
1662
1663         if (key != NULL)
1664                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1665                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1666
1667         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1668         hf = 0;
1669         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1670         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1671         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1672         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1673         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1674         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1675         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1676         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1677         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1678
1679         rss_conf->rss_hf = hf;
1680
1681         return 0;
1682 }
1683
1684 static void
1685 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1686 {
1687         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1688         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1689
1690         /* Bind all local non-queue interrupt to vector 0 */
1691         int_map |= 0;
1692
1693         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1694         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1695         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1696         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1697         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1698         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1699
1700         /* Enable misc causes */
1701         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1702                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1703                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1704                                 FM10K_EIMR_ENABLE(MAILBOX) |
1705                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1706                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1707                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1708                                 FM10K_EIMR_ENABLE(VFLR));
1709
1710         /* Enable ITR 0 */
1711         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1712                                         FM10K_ITR_MASK_CLEAR);
1713         FM10K_WRITE_FLUSH(hw);
1714 }
1715
1716 static void
1717 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1718 {
1719         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1720         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1721
1722         int_map |= 0;
1723
1724         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1725         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1726         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1727         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1728         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1729         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1730
1731         /* Disable misc causes */
1732         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1733                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1734                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1735                                 FM10K_EIMR_DISABLE(MAILBOX) |
1736                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1737                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1738                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1739                                 FM10K_EIMR_DISABLE(VFLR));
1740
1741         /* Disable ITR 0 */
1742         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1743         FM10K_WRITE_FLUSH(hw);
1744 }
1745
1746 static void
1747 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1748 {
1749         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1750         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1751
1752         /* Bind all local non-queue interrupt to vector 0 */
1753         int_map |= 0;
1754
1755         /* Only INT 0 available, other 15 are reserved. */
1756         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1757
1758         /* Enable ITR 0 */
1759         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1760                                         FM10K_ITR_MASK_CLEAR);
1761         FM10K_WRITE_FLUSH(hw);
1762 }
1763
1764 static void
1765 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1766 {
1767         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1768         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1769
1770         int_map |= 0;
1771
1772         /* Only INT 0 available, other 15 are reserved. */
1773         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1774
1775         /* Disable ITR 0 */
1776         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1777         FM10K_WRITE_FLUSH(hw);
1778 }
1779
1780 static int
1781 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1782 {
1783         struct fm10k_fault fault;
1784         int err;
1785         const char *estr = "Unknown error";
1786
1787         /* Process PCA fault */
1788         if (eicr & FM10K_EICR_PCA_FAULT) {
1789                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1790                 if (err)
1791                         goto error;
1792                 switch (fault.type) {
1793                 case PCA_NO_FAULT:
1794                         estr = "PCA_NO_FAULT"; break;
1795                 case PCA_UNMAPPED_ADDR:
1796                         estr = "PCA_UNMAPPED_ADDR"; break;
1797                 case PCA_BAD_QACCESS_PF:
1798                         estr = "PCA_BAD_QACCESS_PF"; break;
1799                 case PCA_BAD_QACCESS_VF:
1800                         estr = "PCA_BAD_QACCESS_VF"; break;
1801                 case PCA_MALICIOUS_REQ:
1802                         estr = "PCA_MALICIOUS_REQ"; break;
1803                 case PCA_POISONED_TLP:
1804                         estr = "PCA_POISONED_TLP"; break;
1805                 case PCA_TLP_ABORT:
1806                         estr = "PCA_TLP_ABORT"; break;
1807                 default:
1808                         goto error;
1809                 }
1810                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1811                         estr, fault.func ? "VF" : "PF", fault.func,
1812                         fault.address, fault.specinfo);
1813         }
1814
1815         /* Process THI fault */
1816         if (eicr & FM10K_EICR_THI_FAULT) {
1817                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1818                 if (err)
1819                         goto error;
1820                 switch (fault.type) {
1821                 case THI_NO_FAULT:
1822                         estr = "THI_NO_FAULT"; break;
1823                 case THI_MAL_DIS_Q_FAULT:
1824                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1825                 default:
1826                         goto error;
1827                 }
1828                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1829                         estr, fault.func ? "VF" : "PF", fault.func,
1830                         fault.address, fault.specinfo);
1831         }
1832
1833         /* Process FUM fault */
1834         if (eicr & FM10K_EICR_FUM_FAULT) {
1835                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1836                 if (err)
1837                         goto error;
1838                 switch (fault.type) {
1839                 case FUM_NO_FAULT:
1840                         estr = "FUM_NO_FAULT"; break;
1841                 case FUM_UNMAPPED_ADDR:
1842                         estr = "FUM_UNMAPPED_ADDR"; break;
1843                 case FUM_POISONED_TLP:
1844                         estr = "FUM_POISONED_TLP"; break;
1845                 case FUM_BAD_VF_QACCESS:
1846                         estr = "FUM_BAD_VF_QACCESS"; break;
1847                 case FUM_ADD_DECODE_ERR:
1848                         estr = "FUM_ADD_DECODE_ERR"; break;
1849                 case FUM_RO_ERROR:
1850                         estr = "FUM_RO_ERROR"; break;
1851                 case FUM_QPRC_CRC_ERROR:
1852                         estr = "FUM_QPRC_CRC_ERROR"; break;
1853                 case FUM_CSR_TIMEOUT:
1854                         estr = "FUM_CSR_TIMEOUT"; break;
1855                 case FUM_INVALID_TYPE:
1856                         estr = "FUM_INVALID_TYPE"; break;
1857                 case FUM_INVALID_LENGTH:
1858                         estr = "FUM_INVALID_LENGTH"; break;
1859                 case FUM_INVALID_BE:
1860                         estr = "FUM_INVALID_BE"; break;
1861                 case FUM_INVALID_ALIGN:
1862                         estr = "FUM_INVALID_ALIGN"; break;
1863                 default:
1864                         goto error;
1865                 }
1866                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1867                         estr, fault.func ? "VF" : "PF", fault.func,
1868                         fault.address, fault.specinfo);
1869         }
1870
1871         return 0;
1872 error:
1873         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1874         return err;
1875 }
1876
1877 /**
1878  * PF interrupt handler triggered by NIC for handling specific interrupt.
1879  *
1880  * @param handle
1881  *  Pointer to interrupt handle.
1882  * @param param
1883  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1884  *
1885  * @return
1886  *  void
1887  */
1888 static void
1889 fm10k_dev_interrupt_handler_pf(
1890                         __rte_unused struct rte_intr_handle *handle,
1891                         void *param)
1892 {
1893         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1894         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1895         uint32_t cause, status;
1896
1897         if (hw->mac.type != fm10k_mac_pf)
1898                 return;
1899
1900         cause = FM10K_READ_REG(hw, FM10K_EICR);
1901
1902         /* Handle PCI fault cases */
1903         if (cause & FM10K_EICR_FAULT_MASK) {
1904                 PMD_INIT_LOG(ERR, "INT: find fault!");
1905                 fm10k_dev_handle_fault(hw, cause);
1906         }
1907
1908         /* Handle switch up/down */
1909         if (cause & FM10K_EICR_SWITCHNOTREADY)
1910                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1911
1912         if (cause & FM10K_EICR_SWITCHREADY)
1913                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1914
1915         /* Handle mailbox message */
1916         fm10k_mbx_lock(hw);
1917         hw->mbx.ops.process(hw, &hw->mbx);
1918         fm10k_mbx_unlock(hw);
1919
1920         /* Handle SRAM error */
1921         if (cause & FM10K_EICR_SRAMERROR) {
1922                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1923
1924                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1925                 /* Write to clear pending bits */
1926                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1927
1928                 /* Todo: print out error message after shared code  updates */
1929         }
1930
1931         /* Clear these 3 events if having any */
1932         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1933                  FM10K_EICR_SWITCHREADY;
1934         if (cause)
1935                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1936
1937         /* Re-enable interrupt from device side */
1938         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1939                                         FM10K_ITR_MASK_CLEAR);
1940         /* Re-enable interrupt from host side */
1941         rte_intr_enable(&(dev->pci_dev->intr_handle));
1942 }
1943
1944 /**
1945  * VF interrupt handler triggered by NIC for handling specific interrupt.
1946  *
1947  * @param handle
1948  *  Pointer to interrupt handle.
1949  * @param param
1950  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1951  *
1952  * @return
1953  *  void
1954  */
1955 static void
1956 fm10k_dev_interrupt_handler_vf(
1957                         __rte_unused struct rte_intr_handle *handle,
1958                         void *param)
1959 {
1960         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1961         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1962
1963         if (hw->mac.type != fm10k_mac_vf)
1964                 return;
1965
1966         /* Handle mailbox message if lock is acquired */
1967         fm10k_mbx_lock(hw);
1968         hw->mbx.ops.process(hw, &hw->mbx);
1969         fm10k_mbx_unlock(hw);
1970
1971         /* Re-enable interrupt from device side */
1972         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1973                                         FM10K_ITR_MASK_CLEAR);
1974         /* Re-enable interrupt from host side */
1975         rte_intr_enable(&(dev->pci_dev->intr_handle));
1976 }
1977
1978 /* Mailbox message handler in VF */
1979 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1980         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1981         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1982         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1983         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1984 };
1985
1986 /* Mailbox message handler in PF */
1987 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1988         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1989         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1990         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1991         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1992         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1993         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1994         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1995 };
1996
1997 static int
1998 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1999 {
2000         int err;
2001
2002         /* Initialize mailbox lock */
2003         fm10k_mbx_initlock(hw);
2004
2005         /* Replace default message handler with new ones */
2006         if (hw->mac.type == fm10k_mac_pf)
2007                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2008         else
2009                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2010
2011         if (err) {
2012                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2013                                 err);
2014                 return err;
2015         }
2016         /* Connect to SM for PF device or PF for VF device */
2017         return hw->mbx.ops.connect(hw, &hw->mbx);
2018 }
2019
2020 static void
2021 fm10k_close_mbx_service(struct fm10k_hw *hw)
2022 {
2023         /* Disconnect from SM for PF device or PF for VF device */
2024         hw->mbx.ops.disconnect(hw, &hw->mbx);
2025 }
2026
2027 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2028         .dev_configure          = fm10k_dev_configure,
2029         .dev_start              = fm10k_dev_start,
2030         .dev_stop               = fm10k_dev_stop,
2031         .dev_close              = fm10k_dev_close,
2032         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2033         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2034         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2035         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2036         .stats_get              = fm10k_stats_get,
2037         .stats_reset            = fm10k_stats_reset,
2038         .link_update            = fm10k_link_update,
2039         .dev_infos_get          = fm10k_dev_infos_get,
2040         .vlan_filter_set        = fm10k_vlan_filter_set,
2041         .vlan_offload_set       = fm10k_vlan_offload_set,
2042         .mac_addr_add           = fm10k_macaddr_add,
2043         .mac_addr_remove        = fm10k_macaddr_remove,
2044         .rx_queue_start         = fm10k_dev_rx_queue_start,
2045         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2046         .tx_queue_start         = fm10k_dev_tx_queue_start,
2047         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2048         .rx_queue_setup         = fm10k_rx_queue_setup,
2049         .rx_queue_release       = fm10k_rx_queue_release,
2050         .tx_queue_setup         = fm10k_tx_queue_setup,
2051         .tx_queue_release       = fm10k_tx_queue_release,
2052         .reta_update            = fm10k_reta_update,
2053         .reta_query             = fm10k_reta_query,
2054         .rss_hash_update        = fm10k_rss_hash_update,
2055         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2056 };
2057
2058 static int
2059 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2060 {
2061         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2062         int diag;
2063         struct fm10k_macvlan_filter_info *macvlan;
2064
2065         PMD_INIT_FUNC_TRACE();
2066
2067         dev->dev_ops = &fm10k_eth_dev_ops;
2068         dev->rx_pkt_burst = &fm10k_recv_pkts;
2069         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2070
2071         if (dev->data->scattered_rx)
2072                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2073
2074         /* only initialize in the primary process */
2075         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2076                 return 0;
2077
2078         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2079         memset(macvlan, 0, sizeof(*macvlan));
2080         /* Vendor and Device ID need to be set before init of shared code */
2081         memset(hw, 0, sizeof(*hw));
2082         hw->device_id = dev->pci_dev->id.device_id;
2083         hw->vendor_id = dev->pci_dev->id.vendor_id;
2084         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2085         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2086         hw->revision_id = 0;
2087         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2088         if (hw->hw_addr == NULL) {
2089                 PMD_INIT_LOG(ERR, "Bad mem resource."
2090                         " Try to blacklist unused devices.");
2091                 return -EIO;
2092         }
2093
2094         /* Store fm10k_adapter pointer */
2095         hw->back = dev->data->dev_private;
2096
2097         /* Initialize the shared code */
2098         diag = fm10k_init_shared_code(hw);
2099         if (diag != FM10K_SUCCESS) {
2100                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2101                 return -EIO;
2102         }
2103
2104         /*
2105          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2106          * there is no way to get link status without reading BAR4.  Until this
2107          * works, assume we have maximum bandwidth.
2108          * @todo - fix bus info
2109          */
2110         hw->bus_caps.speed = fm10k_bus_speed_8000;
2111         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2112         hw->bus_caps.payload = fm10k_bus_payload_512;
2113         hw->bus.speed = fm10k_bus_speed_8000;
2114         hw->bus.width = fm10k_bus_width_pcie_x8;
2115         hw->bus.payload = fm10k_bus_payload_256;
2116
2117         /* Initialize the hw */
2118         diag = fm10k_init_hw(hw);
2119         if (diag != FM10K_SUCCESS) {
2120                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2121                 return -EIO;
2122         }
2123
2124         /* Initialize MAC address(es) */
2125         dev->data->mac_addrs = rte_zmalloc("fm10k",
2126                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2127         if (dev->data->mac_addrs == NULL) {
2128                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2129                 return -ENOMEM;
2130         }
2131
2132         diag = fm10k_read_mac_addr(hw);
2133
2134         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2135                         &dev->data->mac_addrs[0]);
2136
2137         if (diag != FM10K_SUCCESS ||
2138                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2139
2140                 /* Generate a random addr */
2141                 eth_random_addr(hw->mac.addr);
2142                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2143                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2144                 &dev->data->mac_addrs[0]);
2145         }
2146
2147         /* Reset the hw statistics */
2148         fm10k_stats_reset(dev);
2149
2150         /* Reset the hw */
2151         diag = fm10k_reset_hw(hw);
2152         if (diag != FM10K_SUCCESS) {
2153                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2154                 return -EIO;
2155         }
2156
2157         /* Setup mailbox service */
2158         diag = fm10k_setup_mbx_service(hw);
2159         if (diag != FM10K_SUCCESS) {
2160                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2161                 return -EIO;
2162         }
2163
2164         /*PF/VF has different interrupt handling mechanism */
2165         if (hw->mac.type == fm10k_mac_pf) {
2166                 /* register callback func to eal lib */
2167                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2168                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2169
2170                 /* enable MISC interrupt */
2171                 fm10k_dev_enable_intr_pf(dev);
2172         } else { /* VF */
2173                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2174                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2175
2176                 fm10k_dev_enable_intr_vf(dev);
2177         }
2178
2179         /* Enable uio intr after callback registered */
2180         rte_intr_enable(&(dev->pci_dev->intr_handle));
2181
2182         hw->mac.ops.update_int_moderator(hw);
2183
2184         /* Make sure Switch Manager is ready before going forward. */
2185         if (hw->mac.type == fm10k_mac_pf) {
2186                 int switch_ready = 0;
2187                 int i;
2188
2189                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2190                         fm10k_mbx_lock(hw);
2191                         hw->mac.ops.get_host_state(hw, &switch_ready);
2192                         fm10k_mbx_unlock(hw);
2193                         if (switch_ready)
2194                                 break;
2195                         /* Delay some time to acquire async LPORT_MAP info. */
2196                         rte_delay_us(WAIT_SWITCH_MSG_US);
2197                 }
2198
2199                 if (switch_ready == 0) {
2200                         PMD_INIT_LOG(ERR, "switch is not ready");
2201                         return -1;
2202                 }
2203         }
2204
2205         /*
2206          * Below function will trigger operations on mailbox, acquire lock to
2207          * avoid race condition from interrupt handler. Operations on mailbox
2208          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2209          * will handle and generate an interrupt to our side. Then,  FIFO in
2210          * mailbox will be touched.
2211          */
2212         fm10k_mbx_lock(hw);
2213         /* Enable port first */
2214         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2215
2216         /* Set unicast mode by default. App can change to other mode in other
2217          * API func.
2218          */
2219         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2220                                         FM10K_XCAST_MODE_NONE);
2221
2222         fm10k_mbx_unlock(hw);
2223
2224         /* Add default mac address */
2225         fm10k_MAC_filter_set(dev, hw->mac.addr, true);
2226
2227         return 0;
2228 }
2229
2230 static int
2231 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2232 {
2233         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2234
2235         PMD_INIT_FUNC_TRACE();
2236
2237         /* only uninitialize in the primary process */
2238         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2239                 return 0;
2240
2241         /* safe to close dev here */
2242         fm10k_dev_close(dev);
2243
2244         dev->dev_ops = NULL;
2245         dev->rx_pkt_burst = NULL;
2246         dev->tx_pkt_burst = NULL;
2247
2248         /* disable uio/vfio intr */
2249         rte_intr_disable(&(dev->pci_dev->intr_handle));
2250
2251         /*PF/VF has different interrupt handling mechanism */
2252         if (hw->mac.type == fm10k_mac_pf) {
2253                 /* disable interrupt */
2254                 fm10k_dev_disable_intr_pf(dev);
2255
2256                 /* unregister callback func to eal lib */
2257                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2258                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2259         } else {
2260                 /* disable interrupt */
2261                 fm10k_dev_disable_intr_vf(dev);
2262
2263                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2264                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2265         }
2266
2267         /* free mac memory */
2268         if (dev->data->mac_addrs) {
2269                 rte_free(dev->data->mac_addrs);
2270                 dev->data->mac_addrs = NULL;
2271         }
2272
2273         memset(hw, 0, sizeof(*hw));
2274
2275         return 0;
2276 }
2277
2278 /*
2279  * The set of PCI devices this driver supports. This driver will enable both PF
2280  * and SRIOV-VF devices.
2281  */
2282 static const struct rte_pci_id pci_id_fm10k_map[] = {
2283 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2284 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2285 #include "rte_pci_dev_ids.h"
2286         { .vendor_id = 0, /* sentinel */ },
2287 };
2288
2289 static struct eth_driver rte_pmd_fm10k = {
2290         .pci_drv = {
2291                 .name = "rte_pmd_fm10k",
2292                 .id_table = pci_id_fm10k_map,
2293                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2294         },
2295         .eth_dev_init = eth_fm10k_dev_init,
2296         .eth_dev_uninit = eth_fm10k_dev_uninit,
2297         .dev_private_size = sizeof(struct fm10k_adapter),
2298 };
2299
2300 /*
2301  * Driver initialization routine.
2302  * Invoked once at EAL init time.
2303  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2304  */
2305 static int
2306 rte_pmd_fm10k_init(__rte_unused const char *name,
2307         __rte_unused const char *params)
2308 {
2309         PMD_INIT_FUNC_TRACE();
2310         rte_eth_driver_register(&rte_pmd_fm10k);
2311         return 0;
2312 }
2313
2314 static struct rte_driver rte_fm10k_driver = {
2315         .type = PMD_PDEV,
2316         .init = rte_pmd_fm10k_init,
2317 };
2318
2319 PMD_REGISTER_DRIVER(rte_fm10k_driver);