drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 /* Max try times to acquire switch status */
  49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  50 /* Wait interval to get switch status */
  51 #define WAIT_SWITCH_MSG_US    100000
  52 /* Number of chars per uint32 type */
  53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  55
  56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  62 static int
  63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  64 static void
  65 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add);
  66 static void
  67 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         return 0;
 125 }
 126
 127 /*
 128  * clean queue, descriptor rings, free software buffers used when stopping
 129  * device.
 130  */
 131 static inline void
 132 rx_queue_clean(struct fm10k_rx_queue *q)
 133 {
 134         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 135         uint32_t i;
 136         PMD_INIT_FUNC_TRACE();
 137
 138         /* zero descriptor rings */
 139         for (i = 0; i < q->nb_desc; ++i)
 140                 q->hw_ring[i] = zero;
 141
 142         /* free software buffers */
 143         for (i = 0; i < q->nb_desc; ++i) {
 144                 if (q->sw_ring[i]) {
 145                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 146                         q->sw_ring[i] = NULL;
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * free all queue memory used when releasing the queue (i.e. configure)
 153  */
 154 static inline void
 155 rx_queue_free(struct fm10k_rx_queue *q)
 156 {
 157         PMD_INIT_FUNC_TRACE();
 158         if (q) {
 159                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 160                 rx_queue_clean(q);
 161                 if (q->sw_ring) {
 162                         rte_free(q->sw_ring);
 163                         q->sw_ring = NULL;
 164                 }
 165                 rte_free(q);
 166                 q = NULL;
 167         }
 168 }
 169
 170 /*
 171  * disable RX queue, wait unitl HW finished necessary flush operation
 172  */
 173 static inline int
 174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 175 {
 176         uint32_t reg, i;
 177
 178         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 179         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 180                         reg & ~FM10K_RXQCTL_ENABLE);
 181
 182         /* Wait 100us at most */
 183         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 184                 rte_delay_us(1);
 185                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 186                 if (!(reg & FM10K_RXQCTL_ENABLE))
 187                         break;
 188         }
 189
 190         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 191                 return -1;
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * reset queue to initial state, allocate software buffers used when starting
 198  * device
 199  */
 200 static inline void
 201 tx_queue_reset(struct fm10k_tx_queue *q)
 202 {
 203         PMD_INIT_FUNC_TRACE();
 204         q->last_free = 0;
 205         q->next_free = 0;
 206         q->nb_used = 0;
 207         q->nb_free = q->nb_desc - 1;
 208         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 209         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 210 }
 211
 212 /*
 213  * clean queue, descriptor rings, free software buffers used when stopping
 214  * device
 215  */
 216 static inline void
 217 tx_queue_clean(struct fm10k_tx_queue *q)
 218 {
 219         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 220         uint32_t i;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         /* zero descriptor rings */
 224         for (i = 0; i < q->nb_desc; ++i)
 225                 q->hw_ring[i] = zero;
 226
 227         /* free software buffers */
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 if (q->sw_ring[i]) {
 230                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 231                         q->sw_ring[i] = NULL;
 232                 }
 233         }
 234 }
 235
 236 /*
 237  * free all queue memory used when releasing the queue (i.e. configure)
 238  */
 239 static inline void
 240 tx_queue_free(struct fm10k_tx_queue *q)
 241 {
 242         PMD_INIT_FUNC_TRACE();
 243         if (q) {
 244                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 245                 tx_queue_clean(q);
 246                 if (q->rs_tracker.list) {
 247                         rte_free(q->rs_tracker.list);
 248                         q->rs_tracker.list = NULL;
 249                 }
 250                 if (q->sw_ring) {
 251                         rte_free(q->sw_ring);
 252                         q->sw_ring = NULL;
 253                 }
 254                 rte_free(q);
 255                 q = NULL;
 256         }
 257 }
 258
 259 /*
 260  * disable TX queue, wait unitl HW finished necessary flush operation
 261  */
 262 static inline int
 263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 264 {
 265         uint32_t reg, i;
 266
 267         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 268         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 269                         reg & ~FM10K_TXDCTL_ENABLE);
 270
 271         /* Wait 100us at most */
 272         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 273                 rte_delay_us(1);
 274                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 275                 if (!(reg & FM10K_TXDCTL_ENABLE))
 276                         break;
 277         }
 278
 279         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 280                 return -1;
 281
 282         return 0;
 283 }
 284
 285 static int
 286 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 287 {
 288         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 291         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 292
 293         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 294
 295         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 296                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 297                 return -EINVAL;
 298         }
 299
 300         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 301                 return 0;
 302
 303         if (hw->mac.type == fm10k_mac_vf) {
 304                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 305                 return -EINVAL;
 306         }
 307
 308         /* Check VMDQ queue pool number */
 309         if (vmdq_conf->nb_queue_pools >
 310                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 311                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 312                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 313                         vmdq_conf->nb_queue_pools);
 314                 return -EINVAL;
 315         }
 316
 317         return 0;
 318 }
 319
 320 static int
 321 fm10k_dev_configure(struct rte_eth_dev *dev)
 322 {
 323         int ret;
 324
 325         PMD_INIT_FUNC_TRACE();
 326
 327         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 328                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 329         /* multipe queue mode checking */
 330         ret  = fm10k_check_mq_mode(dev);
 331         if (ret != 0) {
 332                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 333                             ret);
 334                 return ret;
 335         }
 336
 337         return 0;
 338 }
 339
 340 static void
 341 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 342 {
 343         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 344         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 345         uint32_t mrqc, *key, i, reta, j;
 346         uint64_t hf;
 347
 348 #define RSS_KEY_SIZE 40
 349         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 350                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 351                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 352                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 353                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 354                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 355         };
 356
 357         if (dev->data->nb_rx_queues == 1 ||
 358             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 359             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 360                 return;
 361
 362         /* random key is rss_intel_key (default) or user provided (rss_key) */
 363         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 364                 key = (uint32_t *)rss_intel_key;
 365         else
 366                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 367
 368         /* Now fill our hash function seeds, 4 bytes at a time */
 369         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 370                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 371
 372         /*
 373          * Fill in redirection table
 374          * The byte-swap is needed because NIC registers are in
 375          * little-endian order.
 376          */
 377         reta = 0;
 378         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 379                 if (j == dev->data->nb_rx_queues)
 380                         j = 0;
 381                 reta = (reta << CHAR_BIT) | j;
 382                 if ((i & 3) == 3)
 383                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 384                                         rte_bswap32(reta));
 385         }
 386
 387         /*
 388          * Generate RSS hash based on packet types, TCP/UDP
 389          * port numbers and/or IPv4/v6 src and dst addresses
 390          */
 391         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 392         mrqc = 0;
 393         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 394         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 395         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 396         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 397         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 398         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 399         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 400         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 401         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 402
 403         if (mrqc == 0) {
 404                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 405                         "supported", hf);
 406                 return;
 407         }
 408
 409         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 410 }
 411
 412 static int
 413 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 414 {
 415         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 416         int i, ret;
 417         struct fm10k_tx_queue *txq;
 418         uint64_t base_addr;
 419         uint32_t size;
 420
 421         /* Disable TXINT to avoid possible interrupt */
 422         for (i = 0; i < hw->mac.max_queues; i++)
 423                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 424                                 3 << FM10K_TXINT_TIMER_SHIFT);
 425
 426         /* Setup TX queue */
 427         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 428                 txq = dev->data->tx_queues[i];
 429                 base_addr = txq->hw_ring_phys_addr;
 430                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 431
 432                 /* disable queue to avoid issues while updating state */
 433                 ret = tx_queue_disable(hw, i);
 434                 if (ret) {
 435                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 436                         return -1;
 437                 }
 438
 439                 /* set location and size for descriptor ring */
 440                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 441                                 base_addr & UINT64_LOWER_32BITS_MASK);
 442                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 443                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 444                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 445         }
 446         return 0;
 447 }
 448
 449 static int
 450 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 451 {
 452         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 453         int i, ret;
 454         struct fm10k_rx_queue *rxq;
 455         uint64_t base_addr;
 456         uint32_t size;
 457         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 458         uint16_t buf_size;
 459
 460         /* Disable RXINT to avoid possible interrupt */
 461         for (i = 0; i < hw->mac.max_queues; i++)
 462                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 463                                 3 << FM10K_RXINT_TIMER_SHIFT);
 464
 465         /* Setup RX queues */
 466         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 467                 rxq = dev->data->rx_queues[i];
 468                 base_addr = rxq->hw_ring_phys_addr;
 469                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 470
 471                 /* disable queue to avoid issues while updating state */
 472                 ret = rx_queue_disable(hw, i);
 473                 if (ret) {
 474                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 475                         return -1;
 476                 }
 477
 478                 /* Setup the Base and Length of the Rx Descriptor Ring */
 479                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 480                                 base_addr & UINT64_LOWER_32BITS_MASK);
 481                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 482                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 483                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 484
 485                 /* Configure the Rx buffer size for one buff without split */
 486                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 487                         RTE_PKTMBUF_HEADROOM);
 488                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 489                  * reserved for this purpose, and the worst case could be 511B.
 490                  * But SRR reg assumes all buffers have the same size. In order
 491                  * to fill the gap, we'll have to consider the worst case and
 492                  * assume 512B is reserved. If we don't do so, it's possible
 493                  * for HW to overwrite data to next mbuf.
 494                  */
 495                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 496
 497                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 498                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 499
 500                 /* It adds dual VLAN length for supporting dual VLAN */
 501                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 502                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 503                         dev->data->dev_conf.rxmode.enable_scatter) {
 504                         uint32_t reg;
 505                         dev->data->scattered_rx = 1;
 506                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 507                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 508                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 509                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 510                 }
 511
 512                 /* Enable drop on empty, it's RO for VF */
 513                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 514                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 515
 516                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 517                 FM10K_WRITE_FLUSH(hw);
 518         }
 519
 520         /* Configure RSS if applicable */
 521         fm10k_dev_mq_rx_configure(dev);
 522         return 0;
 523 }
 524
 525 static int
 526 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 527 {
 528         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 529         int err = -1;
 530         uint32_t reg;
 531         struct fm10k_rx_queue *rxq;
 532
 533         PMD_INIT_FUNC_TRACE();
 534
 535         if (rx_queue_id < dev->data->nb_rx_queues) {
 536                 rxq = dev->data->rx_queues[rx_queue_id];
 537                 err = rx_queue_reset(rxq);
 538                 if (err == -ENOMEM) {
 539                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 540                         return err;
 541                 } else if (err == -EINVAL) {
 542                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 543                                 " %d", err);
 544                         return err;
 545                 }
 546
 547                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 548                  * Note: this must be done AFTER the queue is enabled on real
 549                  * hardware, but BEFORE the queue is enabled when using the
 550                  * emulation platform. Do it in both places for now and remove
 551                  * this comment and the following two register writes when the
 552                  * emulation platform is no longer being used.
 553                  */
 554                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 555                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 556
 557                 /* Set PF ownership flag for PF devices */
 558                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 559                 if (hw->mac.type == fm10k_mac_pf)
 560                         reg |= FM10K_RXQCTL_PF;
 561                 reg |= FM10K_RXQCTL_ENABLE;
 562                 /* enable RX queue */
 563                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 564                 FM10K_WRITE_FLUSH(hw);
 565
 566                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 567                  * Note: this must be done AFTER the queue is enabled
 568                  */
 569                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 570                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 571         }
 572
 573         return err;
 574 }
 575
 576 static int
 577 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 578 {
 579         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 580
 581         PMD_INIT_FUNC_TRACE();
 582
 583         if (rx_queue_id < dev->data->nb_rx_queues) {
 584                 /* Disable RX queue */
 585                 rx_queue_disable(hw, rx_queue_id);
 586
 587                 /* Free mbuf and clean HW ring */
 588                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 589         }
 590
 591         return 0;
 592 }
 593
 594 static int
 595 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 596 {
 597         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 598         /** @todo - this should be defined in the shared code */
 599 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 600         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 601         int err = 0;
 602
 603         PMD_INIT_FUNC_TRACE();
 604
 605         if (tx_queue_id < dev->data->nb_tx_queues) {
 606                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 607
 608                 /* reset head and tail pointers */
 609                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 610                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 611
 612                 /* enable TX queue */
 613                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 614                                         FM10K_TXDCTL_ENABLE | txdctl);
 615                 FM10K_WRITE_FLUSH(hw);
 616         } else
 617                 err = -1;
 618
 619         return err;
 620 }
 621
 622 static int
 623 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 624 {
 625         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 626
 627         PMD_INIT_FUNC_TRACE();
 628
 629         if (tx_queue_id < dev->data->nb_tx_queues) {
 630                 tx_queue_disable(hw, tx_queue_id);
 631                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 632         }
 633
 634         return 0;
 635 }
 636
 637 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 638 {
 639         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 640                 != FM10K_DGLORTMAP_NONE);
 641 }
 642
 643 static void
 644 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 645 {
 646         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 647         int status;
 648
 649         PMD_INIT_FUNC_TRACE();
 650
 651         /* Return if it didn't acquire valid glort range */
 652         if (!fm10k_glort_valid(hw))
 653                 return;
 654
 655         fm10k_mbx_lock(hw);
 656         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 657                                 FM10K_XCAST_MODE_PROMISC);
 658         fm10k_mbx_unlock(hw);
 659
 660         if (status != FM10K_SUCCESS)
 661                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 662 }
 663
 664 static void
 665 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 666 {
 667         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 668         uint8_t mode;
 669         int status;
 670
 671         PMD_INIT_FUNC_TRACE();
 672
 673         /* Return if it didn't acquire valid glort range */
 674         if (!fm10k_glort_valid(hw))
 675                 return;
 676
 677         if (dev->data->all_multicast == 1)
 678                 mode = FM10K_XCAST_MODE_ALLMULTI;
 679         else
 680                 mode = FM10K_XCAST_MODE_NONE;
 681
 682         fm10k_mbx_lock(hw);
 683         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 684                                 mode);
 685         fm10k_mbx_unlock(hw);
 686
 687         if (status != FM10K_SUCCESS)
 688                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 689 }
 690
 691 static void
 692 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 693 {
 694         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 695         int status;
 696
 697         PMD_INIT_FUNC_TRACE();
 698
 699         /* Return if it didn't acquire valid glort range */
 700         if (!fm10k_glort_valid(hw))
 701                 return;
 702
 703         /* If promiscuous mode is enabled, it doesn't make sense to enable
 704          * allmulticast and disable promiscuous since fm10k only can select
 705          * one of the modes.
 706          */
 707         if (dev->data->promiscuous) {
 708                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 709                         "needn't enable allmulticast");
 710                 return;
 711         }
 712
 713         fm10k_mbx_lock(hw);
 714         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 715                                 FM10K_XCAST_MODE_ALLMULTI);
 716         fm10k_mbx_unlock(hw);
 717
 718         if (status != FM10K_SUCCESS)
 719                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 720 }
 721
 722 static void
 723 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 724 {
 725         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 726         int status;
 727
 728         PMD_INIT_FUNC_TRACE();
 729
 730         /* Return if it didn't acquire valid glort range */
 731         if (!fm10k_glort_valid(hw))
 732                 return;
 733
 734         if (dev->data->promiscuous) {
 735                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 736                         "since promisc mode is enabled");
 737                 return;
 738         }
 739
 740         fm10k_mbx_lock(hw);
 741         /* Change mode to unicast mode */
 742         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 743                                 FM10K_XCAST_MODE_NONE);
 744         fm10k_mbx_unlock(hw);
 745
 746         if (status != FM10K_SUCCESS)
 747                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 748 }
 749
 750 /* fls = find last set bit = 32 minus the number of leading zeros */
 751 #ifndef fls
 752 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 753 #endif
 754 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 755 static int
 756 fm10k_dev_start(struct rte_eth_dev *dev)
 757 {
 758         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 759         int i, diag;
 760
 761         PMD_INIT_FUNC_TRACE();
 762
 763         /* stop, init, then start the hw */
 764         diag = fm10k_stop_hw(hw);
 765         if (diag != FM10K_SUCCESS) {
 766                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 767                 return -EIO;
 768         }
 769
 770         diag = fm10k_init_hw(hw);
 771         if (diag != FM10K_SUCCESS) {
 772                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 773                 return -EIO;
 774         }
 775
 776         diag = fm10k_start_hw(hw);
 777         if (diag != FM10K_SUCCESS) {
 778                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 779                 return -EIO;
 780         }
 781
 782         diag = fm10k_dev_tx_init(dev);
 783         if (diag) {
 784                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 785                 return diag;
 786         }
 787
 788         diag = fm10k_dev_rx_init(dev);
 789         if (diag) {
 790                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 791                 return diag;
 792         }
 793
 794         if (hw->mac.type == fm10k_mac_pf) {
 795                 /* Establish only VSI 0 as valid */
 796                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 797
 798                 /* Configure RSS bits used in RETA table */
 799                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 800                                 fls(dev->data->nb_rx_queues - 1) <<
 801                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 802
 803                 /* Invalidate all other GLORT entries */
 804                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 805                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 806                                         FM10K_DGLORTMAP_NONE);
 807         }
 808
 809         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 810                 struct fm10k_rx_queue *rxq;
 811                 rxq = dev->data->rx_queues[i];
 812
 813                 if (rxq->rx_deferred_start)
 814                         continue;
 815                 diag = fm10k_dev_rx_queue_start(dev, i);
 816                 if (diag != 0) {
 817                         int j;
 818                         for (j = 0; j < i; ++j)
 819                                 rx_queue_clean(dev->data->rx_queues[j]);
 820                         return diag;
 821                 }
 822         }
 823
 824         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 825                 struct fm10k_tx_queue *txq;
 826                 txq = dev->data->tx_queues[i];
 827
 828                 if (txq->tx_deferred_start)
 829                         continue;
 830                 diag = fm10k_dev_tx_queue_start(dev, i);
 831                 if (diag != 0) {
 832                         int j;
 833                         for (j = 0; j < i; ++j)
 834                                 tx_queue_clean(dev->data->tx_queues[j]);
 835                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 836                                 rx_queue_clean(dev->data->rx_queues[j]);
 837                         return diag;
 838                 }
 839         }
 840
 841         /* Update default vlan */
 842         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID)
 843                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 844
 845         return 0;
 846 }
 847
 848 static void
 849 fm10k_dev_stop(struct rte_eth_dev *dev)
 850 {
 851         int i;
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         if (dev->data->tx_queues)
 856                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 857                         fm10k_dev_tx_queue_stop(dev, i);
 858
 859         if (dev->data->rx_queues)
 860                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 861                         fm10k_dev_rx_queue_stop(dev, i);
 862 }
 863
 864 static void
 865 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 866 {
 867         int i;
 868
 869         PMD_INIT_FUNC_TRACE();
 870
 871         if (dev->data->tx_queues) {
 872                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 873                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 874         }
 875
 876         if (dev->data->rx_queues) {
 877                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 878                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 879         }
 880 }
 881
 882 static void
 883 fm10k_dev_close(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886
 887         PMD_INIT_FUNC_TRACE();
 888
 889         fm10k_MACVLAN_remove_all(dev);
 890
 891         /* Stop mailbox service first */
 892         fm10k_close_mbx_service(hw);
 893         fm10k_dev_stop(dev);
 894         fm10k_dev_queue_release(dev);
 895         fm10k_stop_hw(hw);
 896 }
 897
 898 static int
 899 fm10k_link_update(struct rte_eth_dev *dev,
 900         __rte_unused int wait_to_complete)
 901 {
 902         PMD_INIT_FUNC_TRACE();
 903
 904         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 905          * x8 PCIe interface. For now, we leave the speed undefined since there
 906          * is no 50Gbps Ethernet. */
 907         dev->data->dev_link.link_speed  = 0;
 908         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 909         dev->data->dev_link.link_status = 1;
 910
 911         return 0;
 912 }
 913
 914 static void
 915 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 916 {
 917         uint64_t ipackets, opackets, ibytes, obytes;
 918         struct fm10k_hw *hw =
 919                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 920         struct fm10k_hw_stats *hw_stats =
 921                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 922         int i;
 923
 924         PMD_INIT_FUNC_TRACE();
 925
 926         fm10k_update_hw_stats(hw, hw_stats);
 927
 928         ipackets = opackets = ibytes = obytes = 0;
 929         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 930                 (i < hw->mac.max_queues); ++i) {
 931                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 932                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 933                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 934                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 935                 ipackets += stats->q_ipackets[i];
 936                 opackets += stats->q_opackets[i];
 937                 ibytes   += stats->q_ibytes[i];
 938                 obytes   += stats->q_obytes[i];
 939         }
 940         stats->ipackets = ipackets;
 941         stats->opackets = opackets;
 942         stats->ibytes = ibytes;
 943         stats->obytes = obytes;
 944 }
 945
 946 static void
 947 fm10k_stats_reset(struct rte_eth_dev *dev)
 948 {
 949         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 950         struct fm10k_hw_stats *hw_stats =
 951                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 952
 953         PMD_INIT_FUNC_TRACE();
 954
 955         memset(hw_stats, 0, sizeof(*hw_stats));
 956         fm10k_rebind_hw_stats(hw, hw_stats);
 957 }
 958
 959 static void
 960 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 961         struct rte_eth_dev_info *dev_info)
 962 {
 963         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 964
 965         PMD_INIT_FUNC_TRACE();
 966
 967         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 968         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 969         dev_info->max_rx_queues      = hw->mac.max_queues;
 970         dev_info->max_tx_queues      = hw->mac.max_queues;
 971         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
 972         dev_info->max_hash_mac_addrs = 0;
 973         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 974         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 975         dev_info->rx_offload_capa =
 976                 DEV_RX_OFFLOAD_VLAN_STRIP |
 977                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 978                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 979                 DEV_RX_OFFLOAD_TCP_CKSUM;
 980         dev_info->tx_offload_capa =
 981                 DEV_TX_OFFLOAD_VLAN_INSERT |
 982                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
 983                 DEV_TX_OFFLOAD_UDP_CKSUM   |
 984                 DEV_TX_OFFLOAD_TCP_CKSUM   |
 985                 DEV_TX_OFFLOAD_TCP_TSO;
 986
 987         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
 988         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 989
 990         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 991                 .rx_thresh = {
 992                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 993                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 994                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 995                 },
 996                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 997                 .rx_drop_en = 0,
 998         };
 999
1000         dev_info->default_txconf = (struct rte_eth_txconf) {
1001                 .tx_thresh = {
1002                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1003                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1004                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1005                 },
1006                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1007                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1008                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1009                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1010         };
1011
1012 }
1013
1014 static int
1015 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1016 {
1017         s32 result;
1018         uint16_t mac_num = 0;
1019         uint32_t vid_idx, vid_bit, mac_index;
1020         struct fm10k_hw *hw;
1021         struct fm10k_macvlan_filter_info *macvlan;
1022         struct rte_eth_dev_data *data = dev->data;
1023
1024         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1025         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1026
1027         if (vlan_id > ETH_VLAN_ID_MAX) {
1028                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1029                 return (-EINVAL);
1030         }
1031
1032         vid_idx = FM10K_VFTA_IDX(vlan_id);
1033         vid_bit = FM10K_VFTA_BIT(vlan_id);
1034         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1035         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1036                 return 0;
1037         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1038         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1039                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1040                         "in the VLAN filter table");
1041                 return (-EINVAL);
1042         }
1043
1044         fm10k_mbx_lock(hw);
1045         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1046         fm10k_mbx_unlock(hw);
1047         if (result != FM10K_SUCCESS) {
1048                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1049                 return (-EIO);
1050         }
1051
1052         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1053                         (result == FM10K_SUCCESS); mac_index++) {
1054                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1055                         continue;
1056                 if (mac_num > macvlan->mac_num - 1) {
1057                         PMD_INIT_LOG(ERR, "MAC address number "
1058                                         "not match");
1059                         break;
1060                 }
1061                 fm10k_mbx_lock(hw);
1062                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1063                         data->mac_addrs[mac_index].addr_bytes,
1064                         vlan_id, on, 0);
1065                 fm10k_mbx_unlock(hw);
1066                 mac_num++;
1067         }
1068         if (result != FM10K_SUCCESS) {
1069                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1070                 return (-EIO);
1071         }
1072
1073         if (on) {
1074                 macvlan->vlan_num++;
1075                 macvlan->vfta[vid_idx] |= vid_bit;
1076         } else {
1077                 macvlan->vlan_num--;
1078                 macvlan->vfta[vid_idx] &= ~vid_bit;
1079         }
1080         return 0;
1081 }
1082
1083 static void
1084 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1085 {
1086         if (mask & ETH_VLAN_STRIP_MASK) {
1087                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1088                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1089                                         "always on in fm10k");
1090         }
1091
1092         if (mask & ETH_VLAN_EXTEND_MASK) {
1093                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1094                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1095                                         "supported in fm10k");
1096         }
1097
1098         if (mask & ETH_VLAN_FILTER_MASK) {
1099                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1100                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1101         }
1102 }
1103
1104 /* Add/Remove a MAC address, and update filters */
1105 static void
1106 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add)
1107 {
1108         uint32_t i, j, k;
1109         struct fm10k_hw *hw;
1110         struct fm10k_macvlan_filter_info *macvlan;
1111
1112         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1113         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1114
1115         i = 0;
1116         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1117                 if (macvlan->vfta[j]) {
1118                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1119                                 if (macvlan->vfta[j] & (1 << k)) {
1120                                         if (i + 1 > macvlan->vlan_num) {
1121                                                 PMD_INIT_LOG(ERR, "vlan number "
1122                                                                 "not match");
1123                                                 return;
1124                                         }
1125                                         fm10k_mbx_lock(hw);
1126                                         fm10k_update_uc_addr(hw,
1127                                                 hw->mac.dglort_map, mac,
1128                                                 j * FM10K_UINT32_BIT_SIZE + k,
1129                                                 add, 0);
1130                                         fm10k_mbx_unlock(hw);
1131                                         i++;
1132                                 }
1133                         }
1134                 }
1135         }
1136
1137         if (add)
1138                 macvlan->mac_num++;
1139         else
1140                 macvlan->mac_num--;
1141 }
1142
1143 /* Add a MAC address, and update filters */
1144 static void
1145 fm10k_macaddr_add(struct rte_eth_dev *dev,
1146                  struct ether_addr *mac_addr,
1147                  __rte_unused uint32_t index,
1148                  __rte_unused uint32_t pool)
1149 {
1150         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE);
1151 }
1152
1153 /* Remove a MAC address, and update filters */
1154 static void
1155 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1156 {
1157         struct rte_eth_dev_data *data = dev->data;
1158
1159         if (index < FM10K_MAX_MACADDR_NUM)
1160                 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1161                                 FALSE);
1162 }
1163
1164 /* Remove all VLAN and MAC address table entries */
1165 static void
1166 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev)
1167 {
1168         uint32_t j, k;
1169         struct fm10k_macvlan_filter_info *macvlan;
1170
1171         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1172         for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1173                 if (macvlan->vfta[j]) {
1174                         for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1175                                 if (macvlan->vfta[j] & (1 << k))
1176                                         fm10k_vlan_filter_set(dev,
1177                                                 j * FM10K_UINT32_BIT_SIZE + k, false);
1178                         }
1179                 }
1180         }
1181 }
1182
1183 static inline int
1184 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1185 {
1186         if ((request < min) || (request > max) || ((request % mult) != 0))
1187                 return -1;
1188         else
1189                 return 0;
1190 }
1191
1192 /*
1193  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1194  * the physical address is required. If the memzone is already created, then
1195  * this function returns a pointer to the existing memzone.
1196  */
1197 static inline const struct rte_memzone *
1198 allocate_hw_ring(const char *driver_name, const char *ring_name,
1199         uint8_t port_id, uint16_t queue_id, int socket_id,
1200         uint32_t size, uint32_t align)
1201 {
1202         char name[RTE_MEMZONE_NAMESIZE];
1203         const struct rte_memzone *mz;
1204
1205         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1206                  driver_name, ring_name, port_id, queue_id, socket_id);
1207
1208         /* return the memzone if it already exists */
1209         mz = rte_memzone_lookup(name);
1210         if (mz)
1211                 return mz;
1212
1213 #ifdef RTE_LIBRTE_XEN_DOM0
1214         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1215                                            RTE_PGSIZE_2M);
1216 #else
1217         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1218 #endif
1219 }
1220
1221 static inline int
1222 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1223 {
1224         if ((request < min) || (request > max) || ((div % request) != 0))
1225                 return -1;
1226         else
1227                 return 0;
1228 }
1229
1230 static inline int
1231 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1232 {
1233         uint16_t rx_free_thresh;
1234
1235         if (conf->rx_free_thresh == 0)
1236                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1237         else
1238                 rx_free_thresh = conf->rx_free_thresh;
1239
1240         /* make sure the requested threshold satisfies the constraints */
1241         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1242                         FM10K_RX_FREE_THRESH_MAX(q),
1243                         FM10K_RX_FREE_THRESH_DIV(q),
1244                         rx_free_thresh)) {
1245                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1246                         "less than or equal to %u, "
1247                         "greater than or equal to %u, "
1248                         "and a divisor of %u",
1249                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1250                         FM10K_RX_FREE_THRESH_MIN(q),
1251                         FM10K_RX_FREE_THRESH_DIV(q));
1252                 return (-EINVAL);
1253         }
1254
1255         q->alloc_thresh = rx_free_thresh;
1256         q->drop_en = conf->rx_drop_en;
1257         q->rx_deferred_start = conf->rx_deferred_start;
1258
1259         return 0;
1260 }
1261
1262 /*
1263  * Hardware requires specific alignment for Rx packet buffers. At
1264  * least one of the following two conditions must be satisfied.
1265  *  1. Address is 512B aligned
1266  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1267  *
1268  * As such, the driver may need to adjust the DMA address within the
1269  * buffer by up to 512B.
1270  *
1271  * return 1 if the element size is valid, otherwise return 0.
1272  */
1273 static int
1274 mempool_element_size_valid(struct rte_mempool *mp)
1275 {
1276         uint32_t min_size;
1277
1278         /* elt_size includes mbuf header and headroom */
1279         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1280                         RTE_PKTMBUF_HEADROOM;
1281
1282         /* account for up to 512B of alignment */
1283         min_size -= FM10K_RX_DATABUF_ALIGN;
1284
1285         /* sanity check for overflow */
1286         if (min_size > mp->elt_size)
1287                 return 0;
1288
1289         /* size is valid */
1290         return 1;
1291 }
1292
1293 static int
1294 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1295         uint16_t nb_desc, unsigned int socket_id,
1296         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1297 {
1298         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1299         struct fm10k_rx_queue *q;
1300         const struct rte_memzone *mz;
1301
1302         PMD_INIT_FUNC_TRACE();
1303
1304         /* make sure the mempool element size can account for alignment. */
1305         if (!mempool_element_size_valid(mp)) {
1306                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1307                 return (-EINVAL);
1308         }
1309
1310         /* make sure a valid number of descriptors have been requested */
1311         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1312                                 FM10K_MULT_RX_DESC, nb_desc)) {
1313                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1314                         "less than or equal to %"PRIu32", "
1315                         "greater than or equal to %u, "
1316                         "and a multiple of %u",
1317                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1318                         FM10K_MULT_RX_DESC);
1319                 return (-EINVAL);
1320         }
1321
1322         /*
1323          * if this queue existed already, free the associated memory. The
1324          * queue cannot be reused in case we need to allocate memory on
1325          * different socket than was previously used.
1326          */
1327         if (dev->data->rx_queues[queue_id] != NULL) {
1328                 rx_queue_free(dev->data->rx_queues[queue_id]);
1329                 dev->data->rx_queues[queue_id] = NULL;
1330         }
1331
1332         /* allocate memory for the queue structure */
1333         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1334                                 socket_id);
1335         if (q == NULL) {
1336                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1337                 return (-ENOMEM);
1338         }
1339
1340         /* setup queue */
1341         q->mp = mp;
1342         q->nb_desc = nb_desc;
1343         q->port_id = dev->data->port_id;
1344         q->queue_id = queue_id;
1345         q->tail_ptr = (volatile uint32_t *)
1346                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1347         if (handle_rxconf(q, conf))
1348                 return (-EINVAL);
1349
1350         /* allocate memory for the software ring */
1351         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1352                                         nb_desc * sizeof(struct rte_mbuf *),
1353                                         RTE_CACHE_LINE_SIZE, socket_id);
1354         if (q->sw_ring == NULL) {
1355                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1356                 rte_free(q);
1357                 return (-ENOMEM);
1358         }
1359
1360         /*
1361          * allocate memory for the hardware descriptor ring. A memzone large
1362          * enough to hold the maximum ring size is requested to allow for
1363          * resizing in later calls to the queue setup function.
1364          */
1365         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1366                                 dev->data->port_id, queue_id, socket_id,
1367                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1368         if (mz == NULL) {
1369                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1370                 rte_free(q->sw_ring);
1371                 rte_free(q);
1372                 return (-ENOMEM);
1373         }
1374         q->hw_ring = mz->addr;
1375 #ifdef RTE_LIBRTE_XEN_DOM0
1376         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1377 #else
1378         q->hw_ring_phys_addr = mz->phys_addr;
1379 #endif
1380
1381         dev->data->rx_queues[queue_id] = q;
1382         return 0;
1383 }
1384
1385 static void
1386 fm10k_rx_queue_release(void *queue)
1387 {
1388         PMD_INIT_FUNC_TRACE();
1389
1390         rx_queue_free(queue);
1391 }
1392
1393 static inline int
1394 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1395 {
1396         uint16_t tx_free_thresh;
1397         uint16_t tx_rs_thresh;
1398
1399         /* constraint MACROs require that tx_free_thresh is configured
1400          * before tx_rs_thresh */
1401         if (conf->tx_free_thresh == 0)
1402                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1403         else
1404                 tx_free_thresh = conf->tx_free_thresh;
1405
1406         /* make sure the requested threshold satisfies the constraints */
1407         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1408                         FM10K_TX_FREE_THRESH_MAX(q),
1409                         FM10K_TX_FREE_THRESH_DIV(q),
1410                         tx_free_thresh)) {
1411                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1412                         "less than or equal to %u, "
1413                         "greater than or equal to %u, "
1414                         "and a divisor of %u",
1415                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1416                         FM10K_TX_FREE_THRESH_MIN(q),
1417                         FM10K_TX_FREE_THRESH_DIV(q));
1418                 return (-EINVAL);
1419         }
1420
1421         q->free_thresh = tx_free_thresh;
1422
1423         if (conf->tx_rs_thresh == 0)
1424                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1425         else
1426                 tx_rs_thresh = conf->tx_rs_thresh;
1427
1428         q->tx_deferred_start = conf->tx_deferred_start;
1429
1430         /* make sure the requested threshold satisfies the constraints */
1431         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1432                         FM10K_TX_RS_THRESH_MAX(q),
1433                         FM10K_TX_RS_THRESH_DIV(q),
1434                         tx_rs_thresh)) {
1435                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1436                         "less than or equal to %u, "
1437                         "greater than or equal to %u, "
1438                         "and a divisor of %u",
1439                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1440                         FM10K_TX_RS_THRESH_MIN(q),
1441                         FM10K_TX_RS_THRESH_DIV(q));
1442                 return (-EINVAL);
1443         }
1444
1445         q->rs_thresh = tx_rs_thresh;
1446
1447         return 0;
1448 }
1449
1450 static int
1451 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1452         uint16_t nb_desc, unsigned int socket_id,
1453         const struct rte_eth_txconf *conf)
1454 {
1455         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1456         struct fm10k_tx_queue *q;
1457         const struct rte_memzone *mz;
1458
1459         PMD_INIT_FUNC_TRACE();
1460
1461         /* make sure a valid number of descriptors have been requested */
1462         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1463                                 FM10K_MULT_TX_DESC, nb_desc)) {
1464                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1465                         "less than or equal to %"PRIu32", "
1466                         "greater than or equal to %u, "
1467                         "and a multiple of %u",
1468                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1469                         FM10K_MULT_TX_DESC);
1470                 return (-EINVAL);
1471         }
1472
1473         /*
1474          * if this queue existed already, free the associated memory. The
1475          * queue cannot be reused in case we need to allocate memory on
1476          * different socket than was previously used.
1477          */
1478         if (dev->data->tx_queues[queue_id] != NULL) {
1479                 tx_queue_free(dev->data->tx_queues[queue_id]);
1480                 dev->data->tx_queues[queue_id] = NULL;
1481         }
1482
1483         /* allocate memory for the queue structure */
1484         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1485                                 socket_id);
1486         if (q == NULL) {
1487                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1488                 return (-ENOMEM);
1489         }
1490
1491         /* setup queue */
1492         q->nb_desc = nb_desc;
1493         q->port_id = dev->data->port_id;
1494         q->queue_id = queue_id;
1495         q->tail_ptr = (volatile uint32_t *)
1496                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1497         if (handle_txconf(q, conf))
1498                 return (-EINVAL);
1499
1500         /* allocate memory for the software ring */
1501         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1502                                         nb_desc * sizeof(struct rte_mbuf *),
1503                                         RTE_CACHE_LINE_SIZE, socket_id);
1504         if (q->sw_ring == NULL) {
1505                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1506                 rte_free(q);
1507                 return (-ENOMEM);
1508         }
1509
1510         /*
1511          * allocate memory for the hardware descriptor ring. A memzone large
1512          * enough to hold the maximum ring size is requested to allow for
1513          * resizing in later calls to the queue setup function.
1514          */
1515         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1516                                 dev->data->port_id, queue_id, socket_id,
1517                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1518         if (mz == NULL) {
1519                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1520                 rte_free(q->sw_ring);
1521                 rte_free(q);
1522                 return (-ENOMEM);
1523         }
1524         q->hw_ring = mz->addr;
1525 #ifdef RTE_LIBRTE_XEN_DOM0
1526         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1527 #else
1528         q->hw_ring_phys_addr = mz->phys_addr;
1529 #endif
1530
1531         /*
1532          * allocate memory for the RS bit tracker. Enough slots to hold the
1533          * descriptor index for each RS bit needing to be set are required.
1534          */
1535         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1536                                 ((nb_desc + 1) / q->rs_thresh) *
1537                                 sizeof(uint16_t),
1538                                 RTE_CACHE_LINE_SIZE, socket_id);
1539         if (q->rs_tracker.list == NULL) {
1540                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1541                 rte_free(q->sw_ring);
1542                 rte_free(q);
1543                 return (-ENOMEM);
1544         }
1545
1546         dev->data->tx_queues[queue_id] = q;
1547         return 0;
1548 }
1549
1550 static void
1551 fm10k_tx_queue_release(void *queue)
1552 {
1553         PMD_INIT_FUNC_TRACE();
1554
1555         tx_queue_free(queue);
1556 }
1557
1558 static int
1559 fm10k_reta_update(struct rte_eth_dev *dev,
1560                         struct rte_eth_rss_reta_entry64 *reta_conf,
1561                         uint16_t reta_size)
1562 {
1563         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1564         uint16_t i, j, idx, shift;
1565         uint8_t mask;
1566         uint32_t reta;
1567
1568         PMD_INIT_FUNC_TRACE();
1569
1570         if (reta_size > FM10K_MAX_RSS_INDICES) {
1571                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1572                         "(%d) doesn't match the number hardware can supported "
1573                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1574                 return -EINVAL;
1575         }
1576
1577         /*
1578          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1579          * 128-entries in 32 registers
1580          */
1581         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1582                 idx = i / RTE_RETA_GROUP_SIZE;
1583                 shift = i % RTE_RETA_GROUP_SIZE;
1584                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1585                                 BIT_MASK_PER_UINT32);
1586                 if (mask == 0)
1587                         continue;
1588
1589                 reta = 0;
1590                 if (mask != BIT_MASK_PER_UINT32)
1591                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1592
1593                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1594                         if (mask & (0x1 << j)) {
1595                                 if (mask != 0xF)
1596                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1597                                 reta |= reta_conf[idx].reta[shift + j] <<
1598                                                 (CHAR_BIT * j);
1599                         }
1600                 }
1601                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1602         }
1603
1604         return 0;
1605 }
1606
1607 static int
1608 fm10k_reta_query(struct rte_eth_dev *dev,
1609                         struct rte_eth_rss_reta_entry64 *reta_conf,
1610                         uint16_t reta_size)
1611 {
1612         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1613         uint16_t i, j, idx, shift;
1614         uint8_t mask;
1615         uint32_t reta;
1616
1617         PMD_INIT_FUNC_TRACE();
1618
1619         if (reta_size < FM10K_MAX_RSS_INDICES) {
1620                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1621                         "(%d) doesn't match the number hardware can supported "
1622                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1623                 return -EINVAL;
1624         }
1625
1626         /*
1627          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1628          * 128-entries in 32 registers
1629          */
1630         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1631                 idx = i / RTE_RETA_GROUP_SIZE;
1632                 shift = i % RTE_RETA_GROUP_SIZE;
1633                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1634                                 BIT_MASK_PER_UINT32);
1635                 if (mask == 0)
1636                         continue;
1637
1638                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1639                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1640                         if (mask & (0x1 << j))
1641                                 reta_conf[idx].reta[shift + j] = ((reta >>
1642                                         CHAR_BIT * j) & UINT8_MAX);
1643                 }
1644         }
1645
1646         return 0;
1647 }
1648
1649 static int
1650 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1651         struct rte_eth_rss_conf *rss_conf)
1652 {
1653         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1654         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1655         uint32_t mrqc;
1656         uint64_t hf = rss_conf->rss_hf;
1657         int i;
1658
1659         PMD_INIT_FUNC_TRACE();
1660
1661         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1662                 FM10K_RSSRK_ENTRIES_PER_REG)
1663                 return -EINVAL;
1664
1665         if (hf == 0)
1666                 return -EINVAL;
1667
1668         mrqc = 0;
1669         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1670         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1671         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1672         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1673         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1674         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1675         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1676         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1677         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1678
1679         /* If the mapping doesn't fit any supported, return */
1680         if (mrqc == 0)
1681                 return -EINVAL;
1682
1683         if (key != NULL)
1684                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1685                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1686
1687         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1688
1689         return 0;
1690 }
1691
1692 static int
1693 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1694         struct rte_eth_rss_conf *rss_conf)
1695 {
1696         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1697         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1698         uint32_t mrqc;
1699         uint64_t hf;
1700         int i;
1701
1702         PMD_INIT_FUNC_TRACE();
1703
1704         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1705                                 FM10K_RSSRK_ENTRIES_PER_REG)
1706                 return -EINVAL;
1707
1708         if (key != NULL)
1709                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1710                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1711
1712         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1713         hf = 0;
1714         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1715         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1716         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1717         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1718         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1719         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1720         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1721         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1722         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1723
1724         rss_conf->rss_hf = hf;
1725
1726         return 0;
1727 }
1728
1729 static void
1730 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1731 {
1732         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1733         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1734
1735         /* Bind all local non-queue interrupt to vector 0 */
1736         int_map |= 0;
1737
1738         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1739         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1740         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1741         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1742         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1743         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1744
1745         /* Enable misc causes */
1746         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1747                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1748                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1749                                 FM10K_EIMR_ENABLE(MAILBOX) |
1750                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1751                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1752                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1753                                 FM10K_EIMR_ENABLE(VFLR));
1754
1755         /* Enable ITR 0 */
1756         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1757                                         FM10K_ITR_MASK_CLEAR);
1758         FM10K_WRITE_FLUSH(hw);
1759 }
1760
1761 static void
1762 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1763 {
1764         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1765         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1766
1767         int_map |= 0;
1768
1769         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1770         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1771         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1772         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1773         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1774         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1775
1776         /* Disable misc causes */
1777         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1778                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1779                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1780                                 FM10K_EIMR_DISABLE(MAILBOX) |
1781                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1782                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1783                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1784                                 FM10K_EIMR_DISABLE(VFLR));
1785
1786         /* Disable ITR 0 */
1787         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1788         FM10K_WRITE_FLUSH(hw);
1789 }
1790
1791 static void
1792 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1793 {
1794         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1795         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1796
1797         /* Bind all local non-queue interrupt to vector 0 */
1798         int_map |= 0;
1799
1800         /* Only INT 0 available, other 15 are reserved. */
1801         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1802
1803         /* Enable ITR 0 */
1804         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1805                                         FM10K_ITR_MASK_CLEAR);
1806         FM10K_WRITE_FLUSH(hw);
1807 }
1808
1809 static void
1810 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1811 {
1812         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1813         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1814
1815         int_map |= 0;
1816
1817         /* Only INT 0 available, other 15 are reserved. */
1818         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1819
1820         /* Disable ITR 0 */
1821         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1822         FM10K_WRITE_FLUSH(hw);
1823 }
1824
1825 static int
1826 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1827 {
1828         struct fm10k_fault fault;
1829         int err;
1830         const char *estr = "Unknown error";
1831
1832         /* Process PCA fault */
1833         if (eicr & FM10K_EICR_PCA_FAULT) {
1834                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1835                 if (err)
1836                         goto error;
1837                 switch (fault.type) {
1838                 case PCA_NO_FAULT:
1839                         estr = "PCA_NO_FAULT"; break;
1840                 case PCA_UNMAPPED_ADDR:
1841                         estr = "PCA_UNMAPPED_ADDR"; break;
1842                 case PCA_BAD_QACCESS_PF:
1843                         estr = "PCA_BAD_QACCESS_PF"; break;
1844                 case PCA_BAD_QACCESS_VF:
1845                         estr = "PCA_BAD_QACCESS_VF"; break;
1846                 case PCA_MALICIOUS_REQ:
1847                         estr = "PCA_MALICIOUS_REQ"; break;
1848                 case PCA_POISONED_TLP:
1849                         estr = "PCA_POISONED_TLP"; break;
1850                 case PCA_TLP_ABORT:
1851                         estr = "PCA_TLP_ABORT"; break;
1852                 default:
1853                         goto error;
1854                 }
1855                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1856                         estr, fault.func ? "VF" : "PF", fault.func,
1857                         fault.address, fault.specinfo);
1858         }
1859
1860         /* Process THI fault */
1861         if (eicr & FM10K_EICR_THI_FAULT) {
1862                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1863                 if (err)
1864                         goto error;
1865                 switch (fault.type) {
1866                 case THI_NO_FAULT:
1867                         estr = "THI_NO_FAULT"; break;
1868                 case THI_MAL_DIS_Q_FAULT:
1869                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1870                 default:
1871                         goto error;
1872                 }
1873                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1874                         estr, fault.func ? "VF" : "PF", fault.func,
1875                         fault.address, fault.specinfo);
1876         }
1877
1878         /* Process FUM fault */
1879         if (eicr & FM10K_EICR_FUM_FAULT) {
1880                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1881                 if (err)
1882                         goto error;
1883                 switch (fault.type) {
1884                 case FUM_NO_FAULT:
1885                         estr = "FUM_NO_FAULT"; break;
1886                 case FUM_UNMAPPED_ADDR:
1887                         estr = "FUM_UNMAPPED_ADDR"; break;
1888                 case FUM_POISONED_TLP:
1889                         estr = "FUM_POISONED_TLP"; break;
1890                 case FUM_BAD_VF_QACCESS:
1891                         estr = "FUM_BAD_VF_QACCESS"; break;
1892                 case FUM_ADD_DECODE_ERR:
1893                         estr = "FUM_ADD_DECODE_ERR"; break;
1894                 case FUM_RO_ERROR:
1895                         estr = "FUM_RO_ERROR"; break;
1896                 case FUM_QPRC_CRC_ERROR:
1897                         estr = "FUM_QPRC_CRC_ERROR"; break;
1898                 case FUM_CSR_TIMEOUT:
1899                         estr = "FUM_CSR_TIMEOUT"; break;
1900                 case FUM_INVALID_TYPE:
1901                         estr = "FUM_INVALID_TYPE"; break;
1902                 case FUM_INVALID_LENGTH:
1903                         estr = "FUM_INVALID_LENGTH"; break;
1904                 case FUM_INVALID_BE:
1905                         estr = "FUM_INVALID_BE"; break;
1906                 case FUM_INVALID_ALIGN:
1907                         estr = "FUM_INVALID_ALIGN"; break;
1908                 default:
1909                         goto error;
1910                 }
1911                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1912                         estr, fault.func ? "VF" : "PF", fault.func,
1913                         fault.address, fault.specinfo);
1914         }
1915
1916         return 0;
1917 error:
1918         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1919         return err;
1920 }
1921
1922 /**
1923  * PF interrupt handler triggered by NIC for handling specific interrupt.
1924  *
1925  * @param handle
1926  *  Pointer to interrupt handle.
1927  * @param param
1928  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1929  *
1930  * @return
1931  *  void
1932  */
1933 static void
1934 fm10k_dev_interrupt_handler_pf(
1935                         __rte_unused struct rte_intr_handle *handle,
1936                         void *param)
1937 {
1938         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1939         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1940         uint32_t cause, status;
1941
1942         if (hw->mac.type != fm10k_mac_pf)
1943                 return;
1944
1945         cause = FM10K_READ_REG(hw, FM10K_EICR);
1946
1947         /* Handle PCI fault cases */
1948         if (cause & FM10K_EICR_FAULT_MASK) {
1949                 PMD_INIT_LOG(ERR, "INT: find fault!");
1950                 fm10k_dev_handle_fault(hw, cause);
1951         }
1952
1953         /* Handle switch up/down */
1954         if (cause & FM10K_EICR_SWITCHNOTREADY)
1955                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1956
1957         if (cause & FM10K_EICR_SWITCHREADY)
1958                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1959
1960         /* Handle mailbox message */
1961         fm10k_mbx_lock(hw);
1962         hw->mbx.ops.process(hw, &hw->mbx);
1963         fm10k_mbx_unlock(hw);
1964
1965         /* Handle SRAM error */
1966         if (cause & FM10K_EICR_SRAMERROR) {
1967                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1968
1969                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1970                 /* Write to clear pending bits */
1971                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1972
1973                 /* Todo: print out error message after shared code  updates */
1974         }
1975
1976         /* Clear these 3 events if having any */
1977         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1978                  FM10K_EICR_SWITCHREADY;
1979         if (cause)
1980                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1981
1982         /* Re-enable interrupt from device side */
1983         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1984                                         FM10K_ITR_MASK_CLEAR);
1985         /* Re-enable interrupt from host side */
1986         rte_intr_enable(&(dev->pci_dev->intr_handle));
1987 }
1988
1989 /**
1990  * VF interrupt handler triggered by NIC for handling specific interrupt.
1991  *
1992  * @param handle
1993  *  Pointer to interrupt handle.
1994  * @param param
1995  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1996  *
1997  * @return
1998  *  void
1999  */
2000 static void
2001 fm10k_dev_interrupt_handler_vf(
2002                         __rte_unused struct rte_intr_handle *handle,
2003                         void *param)
2004 {
2005         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2006         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2007
2008         if (hw->mac.type != fm10k_mac_vf)
2009                 return;
2010
2011         /* Handle mailbox message if lock is acquired */
2012         fm10k_mbx_lock(hw);
2013         hw->mbx.ops.process(hw, &hw->mbx);
2014         fm10k_mbx_unlock(hw);
2015
2016         /* Re-enable interrupt from device side */
2017         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2018                                         FM10K_ITR_MASK_CLEAR);
2019         /* Re-enable interrupt from host side */
2020         rte_intr_enable(&(dev->pci_dev->intr_handle));
2021 }
2022
2023 /* Mailbox message handler in VF */
2024 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2025         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2026         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2027         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2028         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2029 };
2030
2031 /* Mailbox message handler in PF */
2032 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2033         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2034         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2035         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2036         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2037         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2038         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2039         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2040 };
2041
2042 static int
2043 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2044 {
2045         int err;
2046
2047         /* Initialize mailbox lock */
2048         fm10k_mbx_initlock(hw);
2049
2050         /* Replace default message handler with new ones */
2051         if (hw->mac.type == fm10k_mac_pf)
2052                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2053         else
2054                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2055
2056         if (err) {
2057                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2058                                 err);
2059                 return err;
2060         }
2061         /* Connect to SM for PF device or PF for VF device */
2062         return hw->mbx.ops.connect(hw, &hw->mbx);
2063 }
2064
2065 static void
2066 fm10k_close_mbx_service(struct fm10k_hw *hw)
2067 {
2068         /* Disconnect from SM for PF device or PF for VF device */
2069         hw->mbx.ops.disconnect(hw, &hw->mbx);
2070 }
2071
2072 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2073         .dev_configure          = fm10k_dev_configure,
2074         .dev_start              = fm10k_dev_start,
2075         .dev_stop               = fm10k_dev_stop,
2076         .dev_close              = fm10k_dev_close,
2077         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2078         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2079         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2080         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2081         .stats_get              = fm10k_stats_get,
2082         .stats_reset            = fm10k_stats_reset,
2083         .link_update            = fm10k_link_update,
2084         .dev_infos_get          = fm10k_dev_infos_get,
2085         .vlan_filter_set        = fm10k_vlan_filter_set,
2086         .vlan_offload_set       = fm10k_vlan_offload_set,
2087         .mac_addr_add           = fm10k_macaddr_add,
2088         .mac_addr_remove        = fm10k_macaddr_remove,
2089         .rx_queue_start         = fm10k_dev_rx_queue_start,
2090         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2091         .tx_queue_start         = fm10k_dev_tx_queue_start,
2092         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2093         .rx_queue_setup         = fm10k_rx_queue_setup,
2094         .rx_queue_release       = fm10k_rx_queue_release,
2095         .tx_queue_setup         = fm10k_tx_queue_setup,
2096         .tx_queue_release       = fm10k_tx_queue_release,
2097         .reta_update            = fm10k_reta_update,
2098         .reta_query             = fm10k_reta_query,
2099         .rss_hash_update        = fm10k_rss_hash_update,
2100         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2101 };
2102
2103 static int
2104 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2105 {
2106         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2107         int diag;
2108         struct fm10k_macvlan_filter_info *macvlan;
2109
2110         PMD_INIT_FUNC_TRACE();
2111
2112         dev->dev_ops = &fm10k_eth_dev_ops;
2113         dev->rx_pkt_burst = &fm10k_recv_pkts;
2114         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2115
2116         if (dev->data->scattered_rx)
2117                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2118
2119         /* only initialize in the primary process */
2120         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2121                 return 0;
2122
2123         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2124         memset(macvlan, 0, sizeof(*macvlan));
2125         /* Vendor and Device ID need to be set before init of shared code */
2126         memset(hw, 0, sizeof(*hw));
2127         hw->device_id = dev->pci_dev->id.device_id;
2128         hw->vendor_id = dev->pci_dev->id.vendor_id;
2129         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2130         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2131         hw->revision_id = 0;
2132         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2133         if (hw->hw_addr == NULL) {
2134                 PMD_INIT_LOG(ERR, "Bad mem resource."
2135                         " Try to blacklist unused devices.");
2136                 return -EIO;
2137         }
2138
2139         /* Store fm10k_adapter pointer */
2140         hw->back = dev->data->dev_private;
2141
2142         /* Initialize the shared code */
2143         diag = fm10k_init_shared_code(hw);
2144         if (diag != FM10K_SUCCESS) {
2145                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2146                 return -EIO;
2147         }
2148
2149         /*
2150          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2151          * there is no way to get link status without reading BAR4.  Until this
2152          * works, assume we have maximum bandwidth.
2153          * @todo - fix bus info
2154          */
2155         hw->bus_caps.speed = fm10k_bus_speed_8000;
2156         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2157         hw->bus_caps.payload = fm10k_bus_payload_512;
2158         hw->bus.speed = fm10k_bus_speed_8000;
2159         hw->bus.width = fm10k_bus_width_pcie_x8;
2160         hw->bus.payload = fm10k_bus_payload_256;
2161
2162         /* Initialize the hw */
2163         diag = fm10k_init_hw(hw);
2164         if (diag != FM10K_SUCCESS) {
2165                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2166                 return -EIO;
2167         }
2168
2169         /* Initialize MAC address(es) */
2170         dev->data->mac_addrs = rte_zmalloc("fm10k",
2171                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2172         if (dev->data->mac_addrs == NULL) {
2173                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2174                 return -ENOMEM;
2175         }
2176
2177         diag = fm10k_read_mac_addr(hw);
2178
2179         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2180                         &dev->data->mac_addrs[0]);
2181
2182         if (diag != FM10K_SUCCESS ||
2183                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2184
2185                 /* Generate a random addr */
2186                 eth_random_addr(hw->mac.addr);
2187                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2188                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2189                 &dev->data->mac_addrs[0]);
2190         }
2191
2192         /* Reset the hw statistics */
2193         fm10k_stats_reset(dev);
2194
2195         /* Reset the hw */
2196         diag = fm10k_reset_hw(hw);
2197         if (diag != FM10K_SUCCESS) {
2198                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2199                 return -EIO;
2200         }
2201
2202         /* Setup mailbox service */
2203         diag = fm10k_setup_mbx_service(hw);
2204         if (diag != FM10K_SUCCESS) {
2205                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2206                 return -EIO;
2207         }
2208
2209         /*PF/VF has different interrupt handling mechanism */
2210         if (hw->mac.type == fm10k_mac_pf) {
2211                 /* register callback func to eal lib */
2212                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2213                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2214
2215                 /* enable MISC interrupt */
2216                 fm10k_dev_enable_intr_pf(dev);
2217         } else { /* VF */
2218                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2219                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2220
2221                 fm10k_dev_enable_intr_vf(dev);
2222         }
2223
2224         /* Enable uio intr after callback registered */
2225         rte_intr_enable(&(dev->pci_dev->intr_handle));
2226
2227         hw->mac.ops.update_int_moderator(hw);
2228
2229         /* Make sure Switch Manager is ready before going forward. */
2230         if (hw->mac.type == fm10k_mac_pf) {
2231                 int switch_ready = 0;
2232                 int i;
2233
2234                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2235                         fm10k_mbx_lock(hw);
2236                         hw->mac.ops.get_host_state(hw, &switch_ready);
2237                         fm10k_mbx_unlock(hw);
2238                         if (switch_ready)
2239                                 break;
2240                         /* Delay some time to acquire async LPORT_MAP info. */
2241                         rte_delay_us(WAIT_SWITCH_MSG_US);
2242                 }
2243
2244                 if (switch_ready == 0) {
2245                         PMD_INIT_LOG(ERR, "switch is not ready");
2246                         return -1;
2247                 }
2248         }
2249
2250         /*
2251          * Below function will trigger operations on mailbox, acquire lock to
2252          * avoid race condition from interrupt handler. Operations on mailbox
2253          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2254          * will handle and generate an interrupt to our side. Then,  FIFO in
2255          * mailbox will be touched.
2256          */
2257         fm10k_mbx_lock(hw);
2258         /* Enable port first */
2259         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2260
2261         /* Set unicast mode by default. App can change to other mode in other
2262          * API func.
2263          */
2264         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2265                                         FM10K_XCAST_MODE_NONE);
2266
2267         fm10k_mbx_unlock(hw);
2268
2269         /* Add default mac address */
2270         fm10k_MAC_filter_set(dev, hw->mac.addr, true);
2271
2272         return 0;
2273 }
2274
2275 static int
2276 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2277 {
2278         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2279
2280         PMD_INIT_FUNC_TRACE();
2281
2282         /* only uninitialize in the primary process */
2283         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2284                 return 0;
2285
2286         /* safe to close dev here */
2287         fm10k_dev_close(dev);
2288
2289         dev->dev_ops = NULL;
2290         dev->rx_pkt_burst = NULL;
2291         dev->tx_pkt_burst = NULL;
2292
2293         /* disable uio/vfio intr */
2294         rte_intr_disable(&(dev->pci_dev->intr_handle));
2295
2296         /*PF/VF has different interrupt handling mechanism */
2297         if (hw->mac.type == fm10k_mac_pf) {
2298                 /* disable interrupt */
2299                 fm10k_dev_disable_intr_pf(dev);
2300
2301                 /* unregister callback func to eal lib */
2302                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2303                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2304         } else {
2305                 /* disable interrupt */
2306                 fm10k_dev_disable_intr_vf(dev);
2307
2308                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2309                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2310         }
2311
2312         /* free mac memory */
2313         if (dev->data->mac_addrs) {
2314                 rte_free(dev->data->mac_addrs);
2315                 dev->data->mac_addrs = NULL;
2316         }
2317
2318         memset(hw, 0, sizeof(*hw));
2319
2320         return 0;
2321 }
2322
2323 /*
2324  * The set of PCI devices this driver supports. This driver will enable both PF
2325  * and SRIOV-VF devices.
2326  */
2327 static const struct rte_pci_id pci_id_fm10k_map[] = {
2328 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2329 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2330 #include "rte_pci_dev_ids.h"
2331         { .vendor_id = 0, /* sentinel */ },
2332 };
2333
2334 static struct eth_driver rte_pmd_fm10k = {
2335         .pci_drv = {
2336                 .name = "rte_pmd_fm10k",
2337                 .id_table = pci_id_fm10k_map,
2338                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2339         },
2340         .eth_dev_init = eth_fm10k_dev_init,
2341         .eth_dev_uninit = eth_fm10k_dev_uninit,
2342         .dev_private_size = sizeof(struct fm10k_adapter),
2343 };
2344
2345 /*
2346  * Driver initialization routine.
2347  * Invoked once at EAL init time.
2348  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2349  */
2350 static int
2351 rte_pmd_fm10k_init(__rte_unused const char *name,
2352         __rte_unused const char *params)
2353 {
2354         PMD_INIT_FUNC_TRACE();
2355         rte_eth_driver_register(&rte_pmd_fm10k);
2356         return 0;
2357 }
2358
2359 static struct rte_driver rte_fm10k_driver = {
2360         .type = PMD_PDEV,
2361         .init = rte_pmd_fm10k_init,
2362 };
2363
2364 PMD_REGISTER_DRIVER(rte_fm10k_driver);