drivers/net/fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 /* Default delay to acquire mailbox lock */
  45 #define FM10K_MBXLOCK_DELAY_US 20
  46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  47
  48 #define MAIN_VSI_POOL_NUMBER 0
  49
  50 /* Max try times to acquire switch status */
  51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
  52 /* Wait interval to get switch status */
  53 #define WAIT_SWITCH_MSG_US    100000
  54 /* Number of chars per uint32 type */
  55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  57
  58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
  60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
  61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
  62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
  63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
  64 static int
  65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
  66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
  67         const u8 *mac, bool add, uint32_t pool);
  68 static void fm10k_tx_queue_release(void *queue);
  69 static void fm10k_rx_queue_release(void *queue);
  70
  71 static void
  72 fm10k_mbx_initlock(struct fm10k_hw *hw)
  73 {
  74         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  75 }
  76
  77 static void
  78 fm10k_mbx_lock(struct fm10k_hw *hw)
  79 {
  80         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  81                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  82 }
  83
  84 static void
  85 fm10k_mbx_unlock(struct fm10k_hw *hw)
  86 {
  87         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  88 }
  89
  90 /*
  91  * reset queue to initial state, allocate software buffers used when starting
  92  * device.
  93  * return 0 on success
  94  * return -ENOMEM if buffers cannot be allocated
  95  * return -EINVAL if buffers do not satisfy alignment condition
  96  */
  97 static inline int
  98 rx_queue_reset(struct fm10k_rx_queue *q)
  99 {
 100         uint64_t dma_addr;
 101         int i, diag;
 102         PMD_INIT_FUNC_TRACE();
 103
 104         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
 105         if (diag != 0)
 106                 return -ENOMEM;
 107
 108         for (i = 0; i < q->nb_desc; ++i) {
 109                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
 110                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
 111                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
 112                                                 q->nb_desc);
 113                         return -EINVAL;
 114                 }
 115                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 116                 q->hw_ring[i].q.pkt_addr = dma_addr;
 117                 q->hw_ring[i].q.hdr_addr = dma_addr;
 118         }
 119
 120         q->next_dd = 0;
 121         q->next_alloc = 0;
 122         q->next_trigger = q->alloc_thresh - 1;
 123         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 124         return 0;
 125 }
 126
 127 /*
 128  * clean queue, descriptor rings, free software buffers used when stopping
 129  * device.
 130  */
 131 static inline void
 132 rx_queue_clean(struct fm10k_rx_queue *q)
 133 {
 134         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 135         uint32_t i;
 136         PMD_INIT_FUNC_TRACE();
 137
 138         /* zero descriptor rings */
 139         for (i = 0; i < q->nb_desc; ++i)
 140                 q->hw_ring[i] = zero;
 141
 142         /* free software buffers */
 143         for (i = 0; i < q->nb_desc; ++i) {
 144                 if (q->sw_ring[i]) {
 145                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 146                         q->sw_ring[i] = NULL;
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * free all queue memory used when releasing the queue (i.e. configure)
 153  */
 154 static inline void
 155 rx_queue_free(struct fm10k_rx_queue *q)
 156 {
 157         PMD_INIT_FUNC_TRACE();
 158         if (q) {
 159                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 160                 rx_queue_clean(q);
 161                 if (q->sw_ring) {
 162                         rte_free(q->sw_ring);
 163                         q->sw_ring = NULL;
 164                 }
 165                 rte_free(q);
 166                 q = NULL;
 167         }
 168 }
 169
 170 /*
 171  * disable RX queue, wait unitl HW finished necessary flush operation
 172  */
 173 static inline int
 174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 175 {
 176         uint32_t reg, i;
 177
 178         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 179         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 180                         reg & ~FM10K_RXQCTL_ENABLE);
 181
 182         /* Wait 100us at most */
 183         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 184                 rte_delay_us(1);
 185                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 186                 if (!(reg & FM10K_RXQCTL_ENABLE))
 187                         break;
 188         }
 189
 190         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 191                 return -1;
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * reset queue to initial state, allocate software buffers used when starting
 198  * device
 199  */
 200 static inline void
 201 tx_queue_reset(struct fm10k_tx_queue *q)
 202 {
 203         PMD_INIT_FUNC_TRACE();
 204         q->last_free = 0;
 205         q->next_free = 0;
 206         q->nb_used = 0;
 207         q->nb_free = q->nb_desc - 1;
 208         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 209         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 210 }
 211
 212 /*
 213  * clean queue, descriptor rings, free software buffers used when stopping
 214  * device
 215  */
 216 static inline void
 217 tx_queue_clean(struct fm10k_tx_queue *q)
 218 {
 219         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 220         uint32_t i;
 221         PMD_INIT_FUNC_TRACE();
 222
 223         /* zero descriptor rings */
 224         for (i = 0; i < q->nb_desc; ++i)
 225                 q->hw_ring[i] = zero;
 226
 227         /* free software buffers */
 228         for (i = 0; i < q->nb_desc; ++i) {
 229                 if (q->sw_ring[i]) {
 230                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 231                         q->sw_ring[i] = NULL;
 232                 }
 233         }
 234 }
 235
 236 /*
 237  * free all queue memory used when releasing the queue (i.e. configure)
 238  */
 239 static inline void
 240 tx_queue_free(struct fm10k_tx_queue *q)
 241 {
 242         PMD_INIT_FUNC_TRACE();
 243         if (q) {
 244                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 245                 tx_queue_clean(q);
 246                 if (q->rs_tracker.list) {
 247                         rte_free(q->rs_tracker.list);
 248                         q->rs_tracker.list = NULL;
 249                 }
 250                 if (q->sw_ring) {
 251                         rte_free(q->sw_ring);
 252                         q->sw_ring = NULL;
 253                 }
 254                 rte_free(q);
 255                 q = NULL;
 256         }
 257 }
 258
 259 /*
 260  * disable TX queue, wait unitl HW finished necessary flush operation
 261  */
 262 static inline int
 263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 264 {
 265         uint32_t reg, i;
 266
 267         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 268         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 269                         reg & ~FM10K_TXDCTL_ENABLE);
 270
 271         /* Wait 100us at most */
 272         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 273                 rte_delay_us(1);
 274                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 275                 if (!(reg & FM10K_TXDCTL_ENABLE))
 276                         break;
 277         }
 278
 279         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 280                 return -1;
 281
 282         return 0;
 283 }
 284
 285 static int
 286 fm10k_check_mq_mode(struct rte_eth_dev *dev)
 287 {
 288         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 289         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 290         struct rte_eth_vmdq_rx_conf *vmdq_conf;
 291         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 292
 293         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
 294
 295         if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
 296                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 297                 return -EINVAL;
 298         }
 299
 300         if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
 301                 return 0;
 302
 303         if (hw->mac.type == fm10k_mac_vf) {
 304                 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
 305                 return -EINVAL;
 306         }
 307
 308         /* Check VMDQ queue pool number */
 309         if (vmdq_conf->nb_queue_pools >
 310                         sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
 311                         vmdq_conf->nb_queue_pools > nb_rx_q) {
 312                 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
 313                         vmdq_conf->nb_queue_pools);
 314                 return -EINVAL;
 315         }
 316
 317         return 0;
 318 }
 319
 320 static int
 321 fm10k_dev_configure(struct rte_eth_dev *dev)
 322 {
 323         int ret;
 324
 325         PMD_INIT_FUNC_TRACE();
 326
 327         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 328                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 329         /* multipe queue mode checking */
 330         ret  = fm10k_check_mq_mode(dev);
 331         if (ret != 0) {
 332                 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
 333                             ret);
 334                 return ret;
 335         }
 336
 337         return 0;
 338 }
 339
 340 static void
 341 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 342 {
 343         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 344         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 345         uint32_t mrqc, *key, i, reta, j;
 346         uint64_t hf;
 347
 348 #define RSS_KEY_SIZE 40
 349         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 350                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 351                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 352                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 353                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 354                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 355         };
 356
 357         if (dev->data->nb_rx_queues == 1 ||
 358             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 359             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 360                 return;
 361
 362         /* random key is rss_intel_key (default) or user provided (rss_key) */
 363         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 364                 key = (uint32_t *)rss_intel_key;
 365         else
 366                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 367
 368         /* Now fill our hash function seeds, 4 bytes at a time */
 369         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 370                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 371
 372         /*
 373          * Fill in redirection table
 374          * The byte-swap is needed because NIC registers are in
 375          * little-endian order.
 376          */
 377         reta = 0;
 378         for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
 379                 if (j == dev->data->nb_rx_queues)
 380                         j = 0;
 381                 reta = (reta << CHAR_BIT) | j;
 382                 if ((i & 3) == 3)
 383                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 384                                         rte_bswap32(reta));
 385         }
 386
 387         /*
 388          * Generate RSS hash based on packet types, TCP/UDP
 389          * port numbers and/or IPv4/v6 src and dst addresses
 390          */
 391         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 392         mrqc = 0;
 393         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 394         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 395         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 396         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 397         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 398         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 399         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 400         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 401         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 402
 403         if (mrqc == 0) {
 404                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 405                         "supported", hf);
 406                 return;
 407         }
 408
 409         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 410 }
 411
 412 static int
 413 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 414 {
 415         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 416         int i, ret;
 417         struct fm10k_tx_queue *txq;
 418         uint64_t base_addr;
 419         uint32_t size;
 420
 421         /* Disable TXINT to avoid possible interrupt */
 422         for (i = 0; i < hw->mac.max_queues; i++)
 423                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 424                                 3 << FM10K_TXINT_TIMER_SHIFT);
 425
 426         /* Setup TX queue */
 427         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 428                 txq = dev->data->tx_queues[i];
 429                 base_addr = txq->hw_ring_phys_addr;
 430                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 431
 432                 /* disable queue to avoid issues while updating state */
 433                 ret = tx_queue_disable(hw, i);
 434                 if (ret) {
 435                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 436                         return -1;
 437                 }
 438
 439                 /* set location and size for descriptor ring */
 440                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 441                                 base_addr & UINT64_LOWER_32BITS_MASK);
 442                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 443                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 444                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 445         }
 446         return 0;
 447 }
 448
 449 static int
 450 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 451 {
 452         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 453         int i, ret;
 454         struct fm10k_rx_queue *rxq;
 455         uint64_t base_addr;
 456         uint32_t size;
 457         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 458         uint16_t buf_size;
 459
 460         /* Disable RXINT to avoid possible interrupt */
 461         for (i = 0; i < hw->mac.max_queues; i++)
 462                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 463                                 3 << FM10K_RXINT_TIMER_SHIFT);
 464
 465         /* Setup RX queues */
 466         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 467                 rxq = dev->data->rx_queues[i];
 468                 base_addr = rxq->hw_ring_phys_addr;
 469                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 470
 471                 /* disable queue to avoid issues while updating state */
 472                 ret = rx_queue_disable(hw, i);
 473                 if (ret) {
 474                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 475                         return -1;
 476                 }
 477
 478                 /* Setup the Base and Length of the Rx Descriptor Ring */
 479                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 480                                 base_addr & UINT64_LOWER_32BITS_MASK);
 481                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 482                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 483                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 484
 485                 /* Configure the Rx buffer size for one buff without split */
 486                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 487                         RTE_PKTMBUF_HEADROOM);
 488                 /* As RX buffer is aligned to 512B within mbuf, some bytes are
 489                  * reserved for this purpose, and the worst case could be 511B.
 490                  * But SRR reg assumes all buffers have the same size. In order
 491                  * to fill the gap, we'll have to consider the worst case and
 492                  * assume 512B is reserved. If we don't do so, it's possible
 493                  * for HW to overwrite data to next mbuf.
 494                  */
 495                 buf_size -= FM10K_RX_DATABUF_ALIGN;
 496
 497                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 498                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 499
 500                 /* It adds dual VLAN length for supporting dual VLAN */
 501                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 502                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
 503                         dev->data->dev_conf.rxmode.enable_scatter) {
 504                         uint32_t reg;
 505                         dev->data->scattered_rx = 1;
 506                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 507                         reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
 508                         reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
 509                         FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
 510                 }
 511
 512                 /* Enable drop on empty, it's RO for VF */
 513                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 514                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 515
 516                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 517                 FM10K_WRITE_FLUSH(hw);
 518         }
 519
 520         /* Configure RSS if applicable */
 521         fm10k_dev_mq_rx_configure(dev);
 522         return 0;
 523 }
 524
 525 static int
 526 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 527 {
 528         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 529         int err = -1;
 530         uint32_t reg;
 531         struct fm10k_rx_queue *rxq;
 532
 533         PMD_INIT_FUNC_TRACE();
 534
 535         if (rx_queue_id < dev->data->nb_rx_queues) {
 536                 rxq = dev->data->rx_queues[rx_queue_id];
 537                 err = rx_queue_reset(rxq);
 538                 if (err == -ENOMEM) {
 539                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 540                         return err;
 541                 } else if (err == -EINVAL) {
 542                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 543                                 " %d", err);
 544                         return err;
 545                 }
 546
 547                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 548                  * Note: this must be done AFTER the queue is enabled on real
 549                  * hardware, but BEFORE the queue is enabled when using the
 550                  * emulation platform. Do it in both places for now and remove
 551                  * this comment and the following two register writes when the
 552                  * emulation platform is no longer being used.
 553                  */
 554                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 555                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 556
 557                 /* Set PF ownership flag for PF devices */
 558                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 559                 if (hw->mac.type == fm10k_mac_pf)
 560                         reg |= FM10K_RXQCTL_PF;
 561                 reg |= FM10K_RXQCTL_ENABLE;
 562                 /* enable RX queue */
 563                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 564                 FM10K_WRITE_FLUSH(hw);
 565
 566                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 567                  * Note: this must be done AFTER the queue is enabled
 568                  */
 569                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 570                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 571         }
 572
 573         return err;
 574 }
 575
 576 static int
 577 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 578 {
 579         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 580
 581         PMD_INIT_FUNC_TRACE();
 582
 583         if (rx_queue_id < dev->data->nb_rx_queues) {
 584                 /* Disable RX queue */
 585                 rx_queue_disable(hw, rx_queue_id);
 586
 587                 /* Free mbuf and clean HW ring */
 588                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 589         }
 590
 591         return 0;
 592 }
 593
 594 static int
 595 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 596 {
 597         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 598         /** @todo - this should be defined in the shared code */
 599 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 600         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 601         int err = 0;
 602
 603         PMD_INIT_FUNC_TRACE();
 604
 605         if (tx_queue_id < dev->data->nb_tx_queues) {
 606                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 607
 608                 /* reset head and tail pointers */
 609                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 610                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 611
 612                 /* enable TX queue */
 613                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 614                                         FM10K_TXDCTL_ENABLE | txdctl);
 615                 FM10K_WRITE_FLUSH(hw);
 616         } else
 617                 err = -1;
 618
 619         return err;
 620 }
 621
 622 static int
 623 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 624 {
 625         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 626
 627         PMD_INIT_FUNC_TRACE();
 628
 629         if (tx_queue_id < dev->data->nb_tx_queues) {
 630                 tx_queue_disable(hw, tx_queue_id);
 631                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 632         }
 633
 634         return 0;
 635 }
 636
 637 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
 638 {
 639         return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
 640                 != FM10K_DGLORTMAP_NONE);
 641 }
 642
 643 static void
 644 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
 645 {
 646         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 647         int status;
 648
 649         PMD_INIT_FUNC_TRACE();
 650
 651         /* Return if it didn't acquire valid glort range */
 652         if (!fm10k_glort_valid(hw))
 653                 return;
 654
 655         fm10k_mbx_lock(hw);
 656         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 657                                 FM10K_XCAST_MODE_PROMISC);
 658         fm10k_mbx_unlock(hw);
 659
 660         if (status != FM10K_SUCCESS)
 661                 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
 662 }
 663
 664 static void
 665 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
 666 {
 667         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 668         uint8_t mode;
 669         int status;
 670
 671         PMD_INIT_FUNC_TRACE();
 672
 673         /* Return if it didn't acquire valid glort range */
 674         if (!fm10k_glort_valid(hw))
 675                 return;
 676
 677         if (dev->data->all_multicast == 1)
 678                 mode = FM10K_XCAST_MODE_ALLMULTI;
 679         else
 680                 mode = FM10K_XCAST_MODE_NONE;
 681
 682         fm10k_mbx_lock(hw);
 683         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 684                                 mode);
 685         fm10k_mbx_unlock(hw);
 686
 687         if (status != FM10K_SUCCESS)
 688                 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
 689 }
 690
 691 static void
 692 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
 693 {
 694         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 695         int status;
 696
 697         PMD_INIT_FUNC_TRACE();
 698
 699         /* Return if it didn't acquire valid glort range */
 700         if (!fm10k_glort_valid(hw))
 701                 return;
 702
 703         /* If promiscuous mode is enabled, it doesn't make sense to enable
 704          * allmulticast and disable promiscuous since fm10k only can select
 705          * one of the modes.
 706          */
 707         if (dev->data->promiscuous) {
 708                 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
 709                         "needn't enable allmulticast");
 710                 return;
 711         }
 712
 713         fm10k_mbx_lock(hw);
 714         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 715                                 FM10K_XCAST_MODE_ALLMULTI);
 716         fm10k_mbx_unlock(hw);
 717
 718         if (status != FM10K_SUCCESS)
 719                 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
 720 }
 721
 722 static void
 723 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
 724 {
 725         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 726         int status;
 727
 728         PMD_INIT_FUNC_TRACE();
 729
 730         /* Return if it didn't acquire valid glort range */
 731         if (!fm10k_glort_valid(hw))
 732                 return;
 733
 734         if (dev->data->promiscuous) {
 735                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
 736                         "since promisc mode is enabled");
 737                 return;
 738         }
 739
 740         fm10k_mbx_lock(hw);
 741         /* Change mode to unicast mode */
 742         status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
 743                                 FM10K_XCAST_MODE_NONE);
 744         fm10k_mbx_unlock(hw);
 745
 746         if (status != FM10K_SUCCESS)
 747                 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
 748 }
 749
 750 /* fls = find last set bit = 32 minus the number of leading zeros */
 751 #ifndef fls
 752 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 753 #endif
 754 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 755 static int
 756 fm10k_dev_start(struct rte_eth_dev *dev)
 757 {
 758         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 759         int i, diag;
 760
 761         PMD_INIT_FUNC_TRACE();
 762
 763         /* stop, init, then start the hw */
 764         diag = fm10k_stop_hw(hw);
 765         if (diag != FM10K_SUCCESS) {
 766                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 767                 return -EIO;
 768         }
 769
 770         diag = fm10k_init_hw(hw);
 771         if (diag != FM10K_SUCCESS) {
 772                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 773                 return -EIO;
 774         }
 775
 776         diag = fm10k_start_hw(hw);
 777         if (diag != FM10K_SUCCESS) {
 778                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 779                 return -EIO;
 780         }
 781
 782         diag = fm10k_dev_tx_init(dev);
 783         if (diag) {
 784                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 785                 return diag;
 786         }
 787
 788         diag = fm10k_dev_rx_init(dev);
 789         if (diag) {
 790                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 791                 return diag;
 792         }
 793
 794         if (hw->mac.type == fm10k_mac_pf) {
 795                 /* Establish only VSI 0 as valid */
 796                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 797
 798                 /* Configure RSS bits used in RETA table */
 799                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 800                                 fls(dev->data->nb_rx_queues - 1) <<
 801                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 802
 803                 /* Invalidate all other GLORT entries */
 804                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 805                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 806                                         FM10K_DGLORTMAP_NONE);
 807         }
 808
 809         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 810                 struct fm10k_rx_queue *rxq;
 811                 rxq = dev->data->rx_queues[i];
 812
 813                 if (rxq->rx_deferred_start)
 814                         continue;
 815                 diag = fm10k_dev_rx_queue_start(dev, i);
 816                 if (diag != 0) {
 817                         int j;
 818                         for (j = 0; j < i; ++j)
 819                                 rx_queue_clean(dev->data->rx_queues[j]);
 820                         return diag;
 821                 }
 822         }
 823
 824         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 825                 struct fm10k_tx_queue *txq;
 826                 txq = dev->data->tx_queues[i];
 827
 828                 if (txq->tx_deferred_start)
 829                         continue;
 830                 diag = fm10k_dev_tx_queue_start(dev, i);
 831                 if (diag != 0) {
 832                         int j;
 833                         for (j = 0; j < i; ++j)
 834                                 tx_queue_clean(dev->data->tx_queues[j]);
 835                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 836                                 rx_queue_clean(dev->data->rx_queues[j]);
 837                         return diag;
 838                 }
 839         }
 840
 841         /* Update default vlan */
 842         if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID)
 843                 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
 844
 845         return 0;
 846 }
 847
 848 static void
 849 fm10k_dev_stop(struct rte_eth_dev *dev)
 850 {
 851         int i;
 852
 853         PMD_INIT_FUNC_TRACE();
 854
 855         if (dev->data->tx_queues)
 856                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 857                         fm10k_dev_tx_queue_stop(dev, i);
 858
 859         if (dev->data->rx_queues)
 860                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 861                         fm10k_dev_rx_queue_stop(dev, i);
 862 }
 863
 864 static void
 865 fm10k_dev_queue_release(struct rte_eth_dev *dev)
 866 {
 867         int i;
 868
 869         PMD_INIT_FUNC_TRACE();
 870
 871         if (dev->data->tx_queues) {
 872                 for (i = 0; i < dev->data->nb_tx_queues; i++)
 873                         fm10k_tx_queue_release(dev->data->tx_queues[i]);
 874         }
 875
 876         if (dev->data->rx_queues) {
 877                 for (i = 0; i < dev->data->nb_rx_queues; i++)
 878                         fm10k_rx_queue_release(dev->data->rx_queues[i]);
 879         }
 880 }
 881
 882 static void
 883 fm10k_dev_close(struct rte_eth_dev *dev)
 884 {
 885         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 886         uint16_t nb_lport;
 887         struct fm10k_macvlan_filter_info *macvlan;
 888
 889         PMD_INIT_FUNC_TRACE();
 890
 891         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
 892         nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
 893         fm10k_mbx_lock(hw);
 894         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
 895                 nb_lport, false);
 896         fm10k_mbx_unlock(hw);
 897
 898         /* Stop mailbox service first */
 899         fm10k_close_mbx_service(hw);
 900         fm10k_dev_stop(dev);
 901         fm10k_dev_queue_release(dev);
 902         fm10k_stop_hw(hw);
 903 }
 904
 905 static int
 906 fm10k_link_update(struct rte_eth_dev *dev,
 907         __rte_unused int wait_to_complete)
 908 {
 909         PMD_INIT_FUNC_TRACE();
 910
 911         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 912          * x8 PCIe interface. For now, we leave the speed undefined since there
 913          * is no 50Gbps Ethernet. */
 914         dev->data->dev_link.link_speed  = 0;
 915         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 916         dev->data->dev_link.link_status = 1;
 917
 918         return 0;
 919 }
 920
 921 static void
 922 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 923 {
 924         uint64_t ipackets, opackets, ibytes, obytes;
 925         struct fm10k_hw *hw =
 926                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 927         struct fm10k_hw_stats *hw_stats =
 928                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 929         int i;
 930
 931         PMD_INIT_FUNC_TRACE();
 932
 933         fm10k_update_hw_stats(hw, hw_stats);
 934
 935         ipackets = opackets = ibytes = obytes = 0;
 936         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 937                 (i < hw->mac.max_queues); ++i) {
 938                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 939                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 940                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 941                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 942                 ipackets += stats->q_ipackets[i];
 943                 opackets += stats->q_opackets[i];
 944                 ibytes   += stats->q_ibytes[i];
 945                 obytes   += stats->q_obytes[i];
 946         }
 947         stats->ipackets = ipackets;
 948         stats->opackets = opackets;
 949         stats->ibytes = ibytes;
 950         stats->obytes = obytes;
 951 }
 952
 953 static void
 954 fm10k_stats_reset(struct rte_eth_dev *dev)
 955 {
 956         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 957         struct fm10k_hw_stats *hw_stats =
 958                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 959
 960         PMD_INIT_FUNC_TRACE();
 961
 962         memset(hw_stats, 0, sizeof(*hw_stats));
 963         fm10k_rebind_hw_stats(hw, hw_stats);
 964 }
 965
 966 static void
 967 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 968         struct rte_eth_dev_info *dev_info)
 969 {
 970         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 971
 972         PMD_INIT_FUNC_TRACE();
 973
 974         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 975         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 976         dev_info->max_rx_queues      = hw->mac.max_queues;
 977         dev_info->max_tx_queues      = hw->mac.max_queues;
 978         dev_info->max_mac_addrs      = FM10K_MAX_MACADDR_NUM;
 979         dev_info->max_hash_mac_addrs = 0;
 980         dev_info->max_vfs            = dev->pci_dev->max_vfs;
 981         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 982         dev_info->rx_offload_capa =
 983                 DEV_RX_OFFLOAD_VLAN_STRIP |
 984                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 985                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 986                 DEV_RX_OFFLOAD_TCP_CKSUM;
 987         dev_info->tx_offload_capa =
 988                 DEV_TX_OFFLOAD_VLAN_INSERT |
 989                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
 990                 DEV_TX_OFFLOAD_UDP_CKSUM   |
 991                 DEV_TX_OFFLOAD_TCP_CKSUM   |
 992                 DEV_TX_OFFLOAD_TCP_TSO;
 993
 994         dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
 995         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 996
 997         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 998                 .rx_thresh = {
 999                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1000                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1001                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1002                 },
1003                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1004                 .rx_drop_en = 0,
1005         };
1006
1007         dev_info->default_txconf = (struct rte_eth_txconf) {
1008                 .tx_thresh = {
1009                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1010                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1011                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1012                 },
1013                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1014                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1015                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1016                                 ETH_TXQ_FLAGS_NOOFFLOADS,
1017         };
1018
1019 }
1020
1021 static int
1022 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1023 {
1024         s32 result;
1025         uint16_t mac_num = 0;
1026         uint32_t vid_idx, vid_bit, mac_index;
1027         struct fm10k_hw *hw;
1028         struct fm10k_macvlan_filter_info *macvlan;
1029         struct rte_eth_dev_data *data = dev->data;
1030
1031         hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1032         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1033
1034         if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1035                 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1036                 return (-EINVAL);
1037         }
1038
1039         if (vlan_id > ETH_VLAN_ID_MAX) {
1040                 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1041                 return (-EINVAL);
1042         }
1043
1044         vid_idx = FM10K_VFTA_IDX(vlan_id);
1045         vid_bit = FM10K_VFTA_BIT(vlan_id);
1046         /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1047         if (on && (macvlan->vfta[vid_idx] & vid_bit))
1048                 return 0;
1049         /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1050         if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1051                 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1052                         "in the VLAN filter table");
1053                 return (-EINVAL);
1054         }
1055
1056         fm10k_mbx_lock(hw);
1057         result = fm10k_update_vlan(hw, vlan_id, 0, on);
1058         fm10k_mbx_unlock(hw);
1059         if (result != FM10K_SUCCESS) {
1060                 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1061                 return (-EIO);
1062         }
1063
1064         for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1065                         (result == FM10K_SUCCESS); mac_index++) {
1066                 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1067                         continue;
1068                 if (mac_num > macvlan->mac_num - 1) {
1069                         PMD_INIT_LOG(ERR, "MAC address number "
1070                                         "not match");
1071                         break;
1072                 }
1073                 fm10k_mbx_lock(hw);
1074                 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1075                         data->mac_addrs[mac_index].addr_bytes,
1076                         vlan_id, on, 0);
1077                 fm10k_mbx_unlock(hw);
1078                 mac_num++;
1079         }
1080         if (result != FM10K_SUCCESS) {
1081                 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1082                 return (-EIO);
1083         }
1084
1085         if (on) {
1086                 macvlan->vlan_num++;
1087                 macvlan->vfta[vid_idx] |= vid_bit;
1088         } else {
1089                 macvlan->vlan_num--;
1090                 macvlan->vfta[vid_idx] &= ~vid_bit;
1091         }
1092         return 0;
1093 }
1094
1095 static void
1096 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1097 {
1098         if (mask & ETH_VLAN_STRIP_MASK) {
1099                 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1100                         PMD_INIT_LOG(ERR, "VLAN stripping is "
1101                                         "always on in fm10k");
1102         }
1103
1104         if (mask & ETH_VLAN_EXTEND_MASK) {
1105                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1106                         PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1107                                         "supported in fm10k");
1108         }
1109
1110         if (mask & ETH_VLAN_FILTER_MASK) {
1111                 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1112                         PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1113         }
1114 }
1115
1116 /* Add/Remove a MAC address, and update filters to main VSI */
1117 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1118                 const u8 *mac, bool add, uint32_t pool)
1119 {
1120         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1121         struct fm10k_macvlan_filter_info *macvlan;
1122         uint32_t i, j, k;
1123
1124         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1125
1126         if (pool != MAIN_VSI_POOL_NUMBER) {
1127                 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1128                         "mac to pool %u", pool);
1129                 return;
1130         }
1131         for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1132                 if (!macvlan->vfta[j])
1133                         continue;
1134                 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1135                         if (!(macvlan->vfta[j] & (1 << k)))
1136                                 continue;
1137                         if (i + 1 > macvlan->vlan_num) {
1138                                 PMD_INIT_LOG(ERR, "vlan number not match");
1139                                 return;
1140                         }
1141                         fm10k_mbx_lock(hw);
1142                         fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1143                                 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1144                         fm10k_mbx_unlock(hw);
1145                         i++;
1146                 }
1147         }
1148 }
1149
1150 /* Add/Remove a MAC address, and update filters to VMDQ */
1151 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1152                 const u8 *mac, bool add, uint32_t pool)
1153 {
1154         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1155         struct fm10k_macvlan_filter_info *macvlan;
1156         struct rte_eth_vmdq_rx_conf *vmdq_conf;
1157         uint32_t i;
1158
1159         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1160         vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1161
1162         if (pool > macvlan->nb_queue_pools) {
1163                 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1164                         " Max pool is %u",
1165                         pool, macvlan->nb_queue_pools);
1166                 return;
1167         }
1168         for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1169                 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1170                         continue;
1171                 fm10k_mbx_lock(hw);
1172                 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1173                         vmdq_conf->pool_map[i].vlan_id, add, 0);
1174                 fm10k_mbx_unlock(hw);
1175         }
1176 }
1177
1178 /* Add/Remove a MAC address, and update filters */
1179 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1180                 const u8 *mac, bool add, uint32_t pool)
1181 {
1182         struct fm10k_macvlan_filter_info *macvlan;
1183
1184         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1185
1186         if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1187                 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1188         else
1189                 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1190
1191         if (add)
1192                 macvlan->mac_num++;
1193         else
1194                 macvlan->mac_num--;
1195 }
1196
1197 /* Add a MAC address, and update filters */
1198 static void
1199 fm10k_macaddr_add(struct rte_eth_dev *dev,
1200                 struct ether_addr *mac_addr,
1201                 uint32_t index,
1202                 uint32_t pool)
1203 {
1204         struct fm10k_macvlan_filter_info *macvlan;
1205
1206         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1207         fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1208         macvlan->mac_vmdq_id[index] = pool;
1209 }
1210
1211 /* Remove a MAC address, and update filters */
1212 static void
1213 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1214 {
1215         struct rte_eth_dev_data *data = dev->data;
1216         struct fm10k_macvlan_filter_info *macvlan;
1217
1218         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1219         fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1220                         FALSE, macvlan->mac_vmdq_id[index]);
1221         macvlan->mac_vmdq_id[index] = 0;
1222 }
1223
1224 static inline int
1225 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1226 {
1227         if ((request < min) || (request > max) || ((request % mult) != 0))
1228                 return -1;
1229         else
1230                 return 0;
1231 }
1232
1233 /*
1234  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1235  * the physical address is required. If the memzone is already created, then
1236  * this function returns a pointer to the existing memzone.
1237  */
1238 static inline const struct rte_memzone *
1239 allocate_hw_ring(const char *driver_name, const char *ring_name,
1240         uint8_t port_id, uint16_t queue_id, int socket_id,
1241         uint32_t size, uint32_t align)
1242 {
1243         char name[RTE_MEMZONE_NAMESIZE];
1244         const struct rte_memzone *mz;
1245
1246         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1247                  driver_name, ring_name, port_id, queue_id, socket_id);
1248
1249         /* return the memzone if it already exists */
1250         mz = rte_memzone_lookup(name);
1251         if (mz)
1252                 return mz;
1253
1254 #ifdef RTE_LIBRTE_XEN_DOM0
1255         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1256                                            RTE_PGSIZE_2M);
1257 #else
1258         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1259 #endif
1260 }
1261
1262 static inline int
1263 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1264 {
1265         if ((request < min) || (request > max) || ((div % request) != 0))
1266                 return -1;
1267         else
1268                 return 0;
1269 }
1270
1271 static inline int
1272 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1273 {
1274         uint16_t rx_free_thresh;
1275
1276         if (conf->rx_free_thresh == 0)
1277                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1278         else
1279                 rx_free_thresh = conf->rx_free_thresh;
1280
1281         /* make sure the requested threshold satisfies the constraints */
1282         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1283                         FM10K_RX_FREE_THRESH_MAX(q),
1284                         FM10K_RX_FREE_THRESH_DIV(q),
1285                         rx_free_thresh)) {
1286                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1287                         "less than or equal to %u, "
1288                         "greater than or equal to %u, "
1289                         "and a divisor of %u",
1290                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1291                         FM10K_RX_FREE_THRESH_MIN(q),
1292                         FM10K_RX_FREE_THRESH_DIV(q));
1293                 return (-EINVAL);
1294         }
1295
1296         q->alloc_thresh = rx_free_thresh;
1297         q->drop_en = conf->rx_drop_en;
1298         q->rx_deferred_start = conf->rx_deferred_start;
1299
1300         return 0;
1301 }
1302
1303 /*
1304  * Hardware requires specific alignment for Rx packet buffers. At
1305  * least one of the following two conditions must be satisfied.
1306  *  1. Address is 512B aligned
1307  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
1308  *
1309  * As such, the driver may need to adjust the DMA address within the
1310  * buffer by up to 512B.
1311  *
1312  * return 1 if the element size is valid, otherwise return 0.
1313  */
1314 static int
1315 mempool_element_size_valid(struct rte_mempool *mp)
1316 {
1317         uint32_t min_size;
1318
1319         /* elt_size includes mbuf header and headroom */
1320         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1321                         RTE_PKTMBUF_HEADROOM;
1322
1323         /* account for up to 512B of alignment */
1324         min_size -= FM10K_RX_DATABUF_ALIGN;
1325
1326         /* sanity check for overflow */
1327         if (min_size > mp->elt_size)
1328                 return 0;
1329
1330         /* size is valid */
1331         return 1;
1332 }
1333
1334 static int
1335 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1336         uint16_t nb_desc, unsigned int socket_id,
1337         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1338 {
1339         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1340         struct fm10k_rx_queue *q;
1341         const struct rte_memzone *mz;
1342
1343         PMD_INIT_FUNC_TRACE();
1344
1345         /* make sure the mempool element size can account for alignment. */
1346         if (!mempool_element_size_valid(mp)) {
1347                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1348                 return (-EINVAL);
1349         }
1350
1351         /* make sure a valid number of descriptors have been requested */
1352         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1353                                 FM10K_MULT_RX_DESC, nb_desc)) {
1354                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1355                         "less than or equal to %"PRIu32", "
1356                         "greater than or equal to %u, "
1357                         "and a multiple of %u",
1358                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1359                         FM10K_MULT_RX_DESC);
1360                 return (-EINVAL);
1361         }
1362
1363         /*
1364          * if this queue existed already, free the associated memory. The
1365          * queue cannot be reused in case we need to allocate memory on
1366          * different socket than was previously used.
1367          */
1368         if (dev->data->rx_queues[queue_id] != NULL) {
1369                 rx_queue_free(dev->data->rx_queues[queue_id]);
1370                 dev->data->rx_queues[queue_id] = NULL;
1371         }
1372
1373         /* allocate memory for the queue structure */
1374         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1375                                 socket_id);
1376         if (q == NULL) {
1377                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1378                 return (-ENOMEM);
1379         }
1380
1381         /* setup queue */
1382         q->mp = mp;
1383         q->nb_desc = nb_desc;
1384         q->port_id = dev->data->port_id;
1385         q->queue_id = queue_id;
1386         q->tail_ptr = (volatile uint32_t *)
1387                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1388         if (handle_rxconf(q, conf))
1389                 return (-EINVAL);
1390
1391         /* allocate memory for the software ring */
1392         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1393                                         nb_desc * sizeof(struct rte_mbuf *),
1394                                         RTE_CACHE_LINE_SIZE, socket_id);
1395         if (q->sw_ring == NULL) {
1396                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1397                 rte_free(q);
1398                 return (-ENOMEM);
1399         }
1400
1401         /*
1402          * allocate memory for the hardware descriptor ring. A memzone large
1403          * enough to hold the maximum ring size is requested to allow for
1404          * resizing in later calls to the queue setup function.
1405          */
1406         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1407                                 dev->data->port_id, queue_id, socket_id,
1408                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1409         if (mz == NULL) {
1410                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1411                 rte_free(q->sw_ring);
1412                 rte_free(q);
1413                 return (-ENOMEM);
1414         }
1415         q->hw_ring = mz->addr;
1416 #ifdef RTE_LIBRTE_XEN_DOM0
1417         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1418 #else
1419         q->hw_ring_phys_addr = mz->phys_addr;
1420 #endif
1421
1422         dev->data->rx_queues[queue_id] = q;
1423         return 0;
1424 }
1425
1426 static void
1427 fm10k_rx_queue_release(void *queue)
1428 {
1429         PMD_INIT_FUNC_TRACE();
1430
1431         rx_queue_free(queue);
1432 }
1433
1434 static inline int
1435 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1436 {
1437         uint16_t tx_free_thresh;
1438         uint16_t tx_rs_thresh;
1439
1440         /* constraint MACROs require that tx_free_thresh is configured
1441          * before tx_rs_thresh */
1442         if (conf->tx_free_thresh == 0)
1443                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1444         else
1445                 tx_free_thresh = conf->tx_free_thresh;
1446
1447         /* make sure the requested threshold satisfies the constraints */
1448         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1449                         FM10K_TX_FREE_THRESH_MAX(q),
1450                         FM10K_TX_FREE_THRESH_DIV(q),
1451                         tx_free_thresh)) {
1452                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1453                         "less than or equal to %u, "
1454                         "greater than or equal to %u, "
1455                         "and a divisor of %u",
1456                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1457                         FM10K_TX_FREE_THRESH_MIN(q),
1458                         FM10K_TX_FREE_THRESH_DIV(q));
1459                 return (-EINVAL);
1460         }
1461
1462         q->free_thresh = tx_free_thresh;
1463
1464         if (conf->tx_rs_thresh == 0)
1465                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1466         else
1467                 tx_rs_thresh = conf->tx_rs_thresh;
1468
1469         q->tx_deferred_start = conf->tx_deferred_start;
1470
1471         /* make sure the requested threshold satisfies the constraints */
1472         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1473                         FM10K_TX_RS_THRESH_MAX(q),
1474                         FM10K_TX_RS_THRESH_DIV(q),
1475                         tx_rs_thresh)) {
1476                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1477                         "less than or equal to %u, "
1478                         "greater than or equal to %u, "
1479                         "and a divisor of %u",
1480                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1481                         FM10K_TX_RS_THRESH_MIN(q),
1482                         FM10K_TX_RS_THRESH_DIV(q));
1483                 return (-EINVAL);
1484         }
1485
1486         q->rs_thresh = tx_rs_thresh;
1487
1488         return 0;
1489 }
1490
1491 static int
1492 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1493         uint16_t nb_desc, unsigned int socket_id,
1494         const struct rte_eth_txconf *conf)
1495 {
1496         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1497         struct fm10k_tx_queue *q;
1498         const struct rte_memzone *mz;
1499
1500         PMD_INIT_FUNC_TRACE();
1501
1502         /* make sure a valid number of descriptors have been requested */
1503         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1504                                 FM10K_MULT_TX_DESC, nb_desc)) {
1505                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1506                         "less than or equal to %"PRIu32", "
1507                         "greater than or equal to %u, "
1508                         "and a multiple of %u",
1509                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1510                         FM10K_MULT_TX_DESC);
1511                 return (-EINVAL);
1512         }
1513
1514         /*
1515          * if this queue existed already, free the associated memory. The
1516          * queue cannot be reused in case we need to allocate memory on
1517          * different socket than was previously used.
1518          */
1519         if (dev->data->tx_queues[queue_id] != NULL) {
1520                 tx_queue_free(dev->data->tx_queues[queue_id]);
1521                 dev->data->tx_queues[queue_id] = NULL;
1522         }
1523
1524         /* allocate memory for the queue structure */
1525         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1526                                 socket_id);
1527         if (q == NULL) {
1528                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1529                 return (-ENOMEM);
1530         }
1531
1532         /* setup queue */
1533         q->nb_desc = nb_desc;
1534         q->port_id = dev->data->port_id;
1535         q->queue_id = queue_id;
1536         q->tail_ptr = (volatile uint32_t *)
1537                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1538         if (handle_txconf(q, conf))
1539                 return (-EINVAL);
1540
1541         /* allocate memory for the software ring */
1542         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1543                                         nb_desc * sizeof(struct rte_mbuf *),
1544                                         RTE_CACHE_LINE_SIZE, socket_id);
1545         if (q->sw_ring == NULL) {
1546                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1547                 rte_free(q);
1548                 return (-ENOMEM);
1549         }
1550
1551         /*
1552          * allocate memory for the hardware descriptor ring. A memzone large
1553          * enough to hold the maximum ring size is requested to allow for
1554          * resizing in later calls to the queue setup function.
1555          */
1556         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1557                                 dev->data->port_id, queue_id, socket_id,
1558                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1559         if (mz == NULL) {
1560                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1561                 rte_free(q->sw_ring);
1562                 rte_free(q);
1563                 return (-ENOMEM);
1564         }
1565         q->hw_ring = mz->addr;
1566 #ifdef RTE_LIBRTE_XEN_DOM0
1567         q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1568 #else
1569         q->hw_ring_phys_addr = mz->phys_addr;
1570 #endif
1571
1572         /*
1573          * allocate memory for the RS bit tracker. Enough slots to hold the
1574          * descriptor index for each RS bit needing to be set are required.
1575          */
1576         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1577                                 ((nb_desc + 1) / q->rs_thresh) *
1578                                 sizeof(uint16_t),
1579                                 RTE_CACHE_LINE_SIZE, socket_id);
1580         if (q->rs_tracker.list == NULL) {
1581                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1582                 rte_free(q->sw_ring);
1583                 rte_free(q);
1584                 return (-ENOMEM);
1585         }
1586
1587         dev->data->tx_queues[queue_id] = q;
1588         return 0;
1589 }
1590
1591 static void
1592 fm10k_tx_queue_release(void *queue)
1593 {
1594         PMD_INIT_FUNC_TRACE();
1595
1596         tx_queue_free(queue);
1597 }
1598
1599 static int
1600 fm10k_reta_update(struct rte_eth_dev *dev,
1601                         struct rte_eth_rss_reta_entry64 *reta_conf,
1602                         uint16_t reta_size)
1603 {
1604         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1605         uint16_t i, j, idx, shift;
1606         uint8_t mask;
1607         uint32_t reta;
1608
1609         PMD_INIT_FUNC_TRACE();
1610
1611         if (reta_size > FM10K_MAX_RSS_INDICES) {
1612                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1613                         "(%d) doesn't match the number hardware can supported "
1614                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1615                 return -EINVAL;
1616         }
1617
1618         /*
1619          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1620          * 128-entries in 32 registers
1621          */
1622         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1623                 idx = i / RTE_RETA_GROUP_SIZE;
1624                 shift = i % RTE_RETA_GROUP_SIZE;
1625                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1626                                 BIT_MASK_PER_UINT32);
1627                 if (mask == 0)
1628                         continue;
1629
1630                 reta = 0;
1631                 if (mask != BIT_MASK_PER_UINT32)
1632                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1633
1634                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1635                         if (mask & (0x1 << j)) {
1636                                 if (mask != 0xF)
1637                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1638                                 reta |= reta_conf[idx].reta[shift + j] <<
1639                                                 (CHAR_BIT * j);
1640                         }
1641                 }
1642                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1643         }
1644
1645         return 0;
1646 }
1647
1648 static int
1649 fm10k_reta_query(struct rte_eth_dev *dev,
1650                         struct rte_eth_rss_reta_entry64 *reta_conf,
1651                         uint16_t reta_size)
1652 {
1653         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1654         uint16_t i, j, idx, shift;
1655         uint8_t mask;
1656         uint32_t reta;
1657
1658         PMD_INIT_FUNC_TRACE();
1659
1660         if (reta_size < FM10K_MAX_RSS_INDICES) {
1661                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1662                         "(%d) doesn't match the number hardware can supported "
1663                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1664                 return -EINVAL;
1665         }
1666
1667         /*
1668          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1669          * 128-entries in 32 registers
1670          */
1671         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1672                 idx = i / RTE_RETA_GROUP_SIZE;
1673                 shift = i % RTE_RETA_GROUP_SIZE;
1674                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1675                                 BIT_MASK_PER_UINT32);
1676                 if (mask == 0)
1677                         continue;
1678
1679                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1680                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1681                         if (mask & (0x1 << j))
1682                                 reta_conf[idx].reta[shift + j] = ((reta >>
1683                                         CHAR_BIT * j) & UINT8_MAX);
1684                 }
1685         }
1686
1687         return 0;
1688 }
1689
1690 static int
1691 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1692         struct rte_eth_rss_conf *rss_conf)
1693 {
1694         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1695         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1696         uint32_t mrqc;
1697         uint64_t hf = rss_conf->rss_hf;
1698         int i;
1699
1700         PMD_INIT_FUNC_TRACE();
1701
1702         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1703                 FM10K_RSSRK_ENTRIES_PER_REG)
1704                 return -EINVAL;
1705
1706         if (hf == 0)
1707                 return -EINVAL;
1708
1709         mrqc = 0;
1710         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1711         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1712         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1713         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1714         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1715         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1716         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1717         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1718         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1719
1720         /* If the mapping doesn't fit any supported, return */
1721         if (mrqc == 0)
1722                 return -EINVAL;
1723
1724         if (key != NULL)
1725                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1726                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1727
1728         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1729
1730         return 0;
1731 }
1732
1733 static int
1734 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1735         struct rte_eth_rss_conf *rss_conf)
1736 {
1737         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1738         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1739         uint32_t mrqc;
1740         uint64_t hf;
1741         int i;
1742
1743         PMD_INIT_FUNC_TRACE();
1744
1745         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1746                                 FM10K_RSSRK_ENTRIES_PER_REG)
1747                 return -EINVAL;
1748
1749         if (key != NULL)
1750                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1751                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1752
1753         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1754         hf = 0;
1755         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1756         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1757         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1758         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1759         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1760         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1761         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1762         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1763         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1764
1765         rss_conf->rss_hf = hf;
1766
1767         return 0;
1768 }
1769
1770 static void
1771 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1772 {
1773         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1774         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1775
1776         /* Bind all local non-queue interrupt to vector 0 */
1777         int_map |= 0;
1778
1779         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1780         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1781         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1782         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1783         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1784         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1785
1786         /* Enable misc causes */
1787         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1788                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1789                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1790                                 FM10K_EIMR_ENABLE(MAILBOX) |
1791                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1792                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1793                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1794                                 FM10K_EIMR_ENABLE(VFLR));
1795
1796         /* Enable ITR 0 */
1797         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1798                                         FM10K_ITR_MASK_CLEAR);
1799         FM10K_WRITE_FLUSH(hw);
1800 }
1801
1802 static void
1803 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1804 {
1805         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1806         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1807
1808         int_map |= 0;
1809
1810         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1811         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1812         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1813         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1814         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1815         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1816
1817         /* Disable misc causes */
1818         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1819                                 FM10K_EIMR_DISABLE(THI_FAULT) |
1820                                 FM10K_EIMR_DISABLE(FUM_FAULT) |
1821                                 FM10K_EIMR_DISABLE(MAILBOX) |
1822                                 FM10K_EIMR_DISABLE(SWITCHREADY) |
1823                                 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1824                                 FM10K_EIMR_DISABLE(SRAMERROR) |
1825                                 FM10K_EIMR_DISABLE(VFLR));
1826
1827         /* Disable ITR 0 */
1828         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1829         FM10K_WRITE_FLUSH(hw);
1830 }
1831
1832 static void
1833 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1834 {
1835         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1836         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1837
1838         /* Bind all local non-queue interrupt to vector 0 */
1839         int_map |= 0;
1840
1841         /* Only INT 0 available, other 15 are reserved. */
1842         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1843
1844         /* Enable ITR 0 */
1845         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1846                                         FM10K_ITR_MASK_CLEAR);
1847         FM10K_WRITE_FLUSH(hw);
1848 }
1849
1850 static void
1851 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1852 {
1853         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1854         uint32_t int_map = FM10K_INT_MAP_DISABLE;
1855
1856         int_map |= 0;
1857
1858         /* Only INT 0 available, other 15 are reserved. */
1859         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1860
1861         /* Disable ITR 0 */
1862         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1863         FM10K_WRITE_FLUSH(hw);
1864 }
1865
1866 static int
1867 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1868 {
1869         struct fm10k_fault fault;
1870         int err;
1871         const char *estr = "Unknown error";
1872
1873         /* Process PCA fault */
1874         if (eicr & FM10K_EICR_PCA_FAULT) {
1875                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1876                 if (err)
1877                         goto error;
1878                 switch (fault.type) {
1879                 case PCA_NO_FAULT:
1880                         estr = "PCA_NO_FAULT"; break;
1881                 case PCA_UNMAPPED_ADDR:
1882                         estr = "PCA_UNMAPPED_ADDR"; break;
1883                 case PCA_BAD_QACCESS_PF:
1884                         estr = "PCA_BAD_QACCESS_PF"; break;
1885                 case PCA_BAD_QACCESS_VF:
1886                         estr = "PCA_BAD_QACCESS_VF"; break;
1887                 case PCA_MALICIOUS_REQ:
1888                         estr = "PCA_MALICIOUS_REQ"; break;
1889                 case PCA_POISONED_TLP:
1890                         estr = "PCA_POISONED_TLP"; break;
1891                 case PCA_TLP_ABORT:
1892                         estr = "PCA_TLP_ABORT"; break;
1893                 default:
1894                         goto error;
1895                 }
1896                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1897                         estr, fault.func ? "VF" : "PF", fault.func,
1898                         fault.address, fault.specinfo);
1899         }
1900
1901         /* Process THI fault */
1902         if (eicr & FM10K_EICR_THI_FAULT) {
1903                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1904                 if (err)
1905                         goto error;
1906                 switch (fault.type) {
1907                 case THI_NO_FAULT:
1908                         estr = "THI_NO_FAULT"; break;
1909                 case THI_MAL_DIS_Q_FAULT:
1910                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1911                 default:
1912                         goto error;
1913                 }
1914                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1915                         estr, fault.func ? "VF" : "PF", fault.func,
1916                         fault.address, fault.specinfo);
1917         }
1918
1919         /* Process FUM fault */
1920         if (eicr & FM10K_EICR_FUM_FAULT) {
1921                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1922                 if (err)
1923                         goto error;
1924                 switch (fault.type) {
1925                 case FUM_NO_FAULT:
1926                         estr = "FUM_NO_FAULT"; break;
1927                 case FUM_UNMAPPED_ADDR:
1928                         estr = "FUM_UNMAPPED_ADDR"; break;
1929                 case FUM_POISONED_TLP:
1930                         estr = "FUM_POISONED_TLP"; break;
1931                 case FUM_BAD_VF_QACCESS:
1932                         estr = "FUM_BAD_VF_QACCESS"; break;
1933                 case FUM_ADD_DECODE_ERR:
1934                         estr = "FUM_ADD_DECODE_ERR"; break;
1935                 case FUM_RO_ERROR:
1936                         estr = "FUM_RO_ERROR"; break;
1937                 case FUM_QPRC_CRC_ERROR:
1938                         estr = "FUM_QPRC_CRC_ERROR"; break;
1939                 case FUM_CSR_TIMEOUT:
1940                         estr = "FUM_CSR_TIMEOUT"; break;
1941                 case FUM_INVALID_TYPE:
1942                         estr = "FUM_INVALID_TYPE"; break;
1943                 case FUM_INVALID_LENGTH:
1944                         estr = "FUM_INVALID_LENGTH"; break;
1945                 case FUM_INVALID_BE:
1946                         estr = "FUM_INVALID_BE"; break;
1947                 case FUM_INVALID_ALIGN:
1948                         estr = "FUM_INVALID_ALIGN"; break;
1949                 default:
1950                         goto error;
1951                 }
1952                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1953                         estr, fault.func ? "VF" : "PF", fault.func,
1954                         fault.address, fault.specinfo);
1955         }
1956
1957         return 0;
1958 error:
1959         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1960         return err;
1961 }
1962
1963 /**
1964  * PF interrupt handler triggered by NIC for handling specific interrupt.
1965  *
1966  * @param handle
1967  *  Pointer to interrupt handle.
1968  * @param param
1969  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1970  *
1971  * @return
1972  *  void
1973  */
1974 static void
1975 fm10k_dev_interrupt_handler_pf(
1976                         __rte_unused struct rte_intr_handle *handle,
1977                         void *param)
1978 {
1979         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1980         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1981         uint32_t cause, status;
1982
1983         if (hw->mac.type != fm10k_mac_pf)
1984                 return;
1985
1986         cause = FM10K_READ_REG(hw, FM10K_EICR);
1987
1988         /* Handle PCI fault cases */
1989         if (cause & FM10K_EICR_FAULT_MASK) {
1990                 PMD_INIT_LOG(ERR, "INT: find fault!");
1991                 fm10k_dev_handle_fault(hw, cause);
1992         }
1993
1994         /* Handle switch up/down */
1995         if (cause & FM10K_EICR_SWITCHNOTREADY)
1996                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1997
1998         if (cause & FM10K_EICR_SWITCHREADY)
1999                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2000
2001         /* Handle mailbox message */
2002         fm10k_mbx_lock(hw);
2003         hw->mbx.ops.process(hw, &hw->mbx);
2004         fm10k_mbx_unlock(hw);
2005
2006         /* Handle SRAM error */
2007         if (cause & FM10K_EICR_SRAMERROR) {
2008                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2009
2010                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2011                 /* Write to clear pending bits */
2012                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2013
2014                 /* Todo: print out error message after shared code  updates */
2015         }
2016
2017         /* Clear these 3 events if having any */
2018         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2019                  FM10K_EICR_SWITCHREADY;
2020         if (cause)
2021                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2022
2023         /* Re-enable interrupt from device side */
2024         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2025                                         FM10K_ITR_MASK_CLEAR);
2026         /* Re-enable interrupt from host side */
2027         rte_intr_enable(&(dev->pci_dev->intr_handle));
2028 }
2029
2030 /**
2031  * VF interrupt handler triggered by NIC for handling specific interrupt.
2032  *
2033  * @param handle
2034  *  Pointer to interrupt handle.
2035  * @param param
2036  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2037  *
2038  * @return
2039  *  void
2040  */
2041 static void
2042 fm10k_dev_interrupt_handler_vf(
2043                         __rte_unused struct rte_intr_handle *handle,
2044                         void *param)
2045 {
2046         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2047         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2048
2049         if (hw->mac.type != fm10k_mac_vf)
2050                 return;
2051
2052         /* Handle mailbox message if lock is acquired */
2053         fm10k_mbx_lock(hw);
2054         hw->mbx.ops.process(hw, &hw->mbx);
2055         fm10k_mbx_unlock(hw);
2056
2057         /* Re-enable interrupt from device side */
2058         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2059                                         FM10K_ITR_MASK_CLEAR);
2060         /* Re-enable interrupt from host side */
2061         rte_intr_enable(&(dev->pci_dev->intr_handle));
2062 }
2063
2064 /* Mailbox message handler in VF */
2065 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2066         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2067         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2068         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2069         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2070 };
2071
2072 /* Mailbox message handler in PF */
2073 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2074         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2075         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2076         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2077         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2078         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2079         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2080         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2081 };
2082
2083 static int
2084 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2085 {
2086         int err;
2087
2088         /* Initialize mailbox lock */
2089         fm10k_mbx_initlock(hw);
2090
2091         /* Replace default message handler with new ones */
2092         if (hw->mac.type == fm10k_mac_pf)
2093                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2094         else
2095                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2096
2097         if (err) {
2098                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2099                                 err);
2100                 return err;
2101         }
2102         /* Connect to SM for PF device or PF for VF device */
2103         return hw->mbx.ops.connect(hw, &hw->mbx);
2104 }
2105
2106 static void
2107 fm10k_close_mbx_service(struct fm10k_hw *hw)
2108 {
2109         /* Disconnect from SM for PF device or PF for VF device */
2110         hw->mbx.ops.disconnect(hw, &hw->mbx);
2111 }
2112
2113 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2114         .dev_configure          = fm10k_dev_configure,
2115         .dev_start              = fm10k_dev_start,
2116         .dev_stop               = fm10k_dev_stop,
2117         .dev_close              = fm10k_dev_close,
2118         .promiscuous_enable     = fm10k_dev_promiscuous_enable,
2119         .promiscuous_disable    = fm10k_dev_promiscuous_disable,
2120         .allmulticast_enable    = fm10k_dev_allmulticast_enable,
2121         .allmulticast_disable   = fm10k_dev_allmulticast_disable,
2122         .stats_get              = fm10k_stats_get,
2123         .stats_reset            = fm10k_stats_reset,
2124         .link_update            = fm10k_link_update,
2125         .dev_infos_get          = fm10k_dev_infos_get,
2126         .vlan_filter_set        = fm10k_vlan_filter_set,
2127         .vlan_offload_set       = fm10k_vlan_offload_set,
2128         .mac_addr_add           = fm10k_macaddr_add,
2129         .mac_addr_remove        = fm10k_macaddr_remove,
2130         .rx_queue_start         = fm10k_dev_rx_queue_start,
2131         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
2132         .tx_queue_start         = fm10k_dev_tx_queue_start,
2133         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
2134         .rx_queue_setup         = fm10k_rx_queue_setup,
2135         .rx_queue_release       = fm10k_rx_queue_release,
2136         .tx_queue_setup         = fm10k_tx_queue_setup,
2137         .tx_queue_release       = fm10k_tx_queue_release,
2138         .reta_update            = fm10k_reta_update,
2139         .reta_query             = fm10k_reta_query,
2140         .rss_hash_update        = fm10k_rss_hash_update,
2141         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
2142 };
2143
2144 static int
2145 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2146 {
2147         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2148         int diag;
2149         struct fm10k_macvlan_filter_info *macvlan;
2150
2151         PMD_INIT_FUNC_TRACE();
2152
2153         dev->dev_ops = &fm10k_eth_dev_ops;
2154         dev->rx_pkt_burst = &fm10k_recv_pkts;
2155         dev->tx_pkt_burst = &fm10k_xmit_pkts;
2156
2157         if (dev->data->scattered_rx)
2158                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2159
2160         /* only initialize in the primary process */
2161         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2162                 return 0;
2163
2164         macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2165         memset(macvlan, 0, sizeof(*macvlan));
2166         /* Vendor and Device ID need to be set before init of shared code */
2167         memset(hw, 0, sizeof(*hw));
2168         hw->device_id = dev->pci_dev->id.device_id;
2169         hw->vendor_id = dev->pci_dev->id.vendor_id;
2170         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2171         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2172         hw->revision_id = 0;
2173         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2174         if (hw->hw_addr == NULL) {
2175                 PMD_INIT_LOG(ERR, "Bad mem resource."
2176                         " Try to blacklist unused devices.");
2177                 return -EIO;
2178         }
2179
2180         /* Store fm10k_adapter pointer */
2181         hw->back = dev->data->dev_private;
2182
2183         /* Initialize the shared code */
2184         diag = fm10k_init_shared_code(hw);
2185         if (diag != FM10K_SUCCESS) {
2186                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2187                 return -EIO;
2188         }
2189
2190         /*
2191          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2192          * there is no way to get link status without reading BAR4.  Until this
2193          * works, assume we have maximum bandwidth.
2194          * @todo - fix bus info
2195          */
2196         hw->bus_caps.speed = fm10k_bus_speed_8000;
2197         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2198         hw->bus_caps.payload = fm10k_bus_payload_512;
2199         hw->bus.speed = fm10k_bus_speed_8000;
2200         hw->bus.width = fm10k_bus_width_pcie_x8;
2201         hw->bus.payload = fm10k_bus_payload_256;
2202
2203         /* Initialize the hw */
2204         diag = fm10k_init_hw(hw);
2205         if (diag != FM10K_SUCCESS) {
2206                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2207                 return -EIO;
2208         }
2209
2210         /* Initialize MAC address(es) */
2211         dev->data->mac_addrs = rte_zmalloc("fm10k",
2212                         ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2213         if (dev->data->mac_addrs == NULL) {
2214                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2215                 return -ENOMEM;
2216         }
2217
2218         diag = fm10k_read_mac_addr(hw);
2219
2220         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2221                         &dev->data->mac_addrs[0]);
2222
2223         if (diag != FM10K_SUCCESS ||
2224                 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2225
2226                 /* Generate a random addr */
2227                 eth_random_addr(hw->mac.addr);
2228                 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2229                 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2230                 &dev->data->mac_addrs[0]);
2231         }
2232
2233         /* Reset the hw statistics */
2234         fm10k_stats_reset(dev);
2235
2236         /* Reset the hw */
2237         diag = fm10k_reset_hw(hw);
2238         if (diag != FM10K_SUCCESS) {
2239                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2240                 return -EIO;
2241         }
2242
2243         /* Setup mailbox service */
2244         diag = fm10k_setup_mbx_service(hw);
2245         if (diag != FM10K_SUCCESS) {
2246                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2247                 return -EIO;
2248         }
2249
2250         /*PF/VF has different interrupt handling mechanism */
2251         if (hw->mac.type == fm10k_mac_pf) {
2252                 /* register callback func to eal lib */
2253                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2254                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2255
2256                 /* enable MISC interrupt */
2257                 fm10k_dev_enable_intr_pf(dev);
2258         } else { /* VF */
2259                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2260                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2261
2262                 fm10k_dev_enable_intr_vf(dev);
2263         }
2264
2265         /* Enable uio intr after callback registered */
2266         rte_intr_enable(&(dev->pci_dev->intr_handle));
2267
2268         hw->mac.ops.update_int_moderator(hw);
2269
2270         /* Make sure Switch Manager is ready before going forward. */
2271         if (hw->mac.type == fm10k_mac_pf) {
2272                 int switch_ready = 0;
2273                 int i;
2274
2275                 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2276                         fm10k_mbx_lock(hw);
2277                         hw->mac.ops.get_host_state(hw, &switch_ready);
2278                         fm10k_mbx_unlock(hw);
2279                         if (switch_ready)
2280                                 break;
2281                         /* Delay some time to acquire async LPORT_MAP info. */
2282                         rte_delay_us(WAIT_SWITCH_MSG_US);
2283                 }
2284
2285                 if (switch_ready == 0) {
2286                         PMD_INIT_LOG(ERR, "switch is not ready");
2287                         return -1;
2288                 }
2289         }
2290
2291         /*
2292          * Below function will trigger operations on mailbox, acquire lock to
2293          * avoid race condition from interrupt handler. Operations on mailbox
2294          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2295          * will handle and generate an interrupt to our side. Then,  FIFO in
2296          * mailbox will be touched.
2297          */
2298         fm10k_mbx_lock(hw);
2299         /* Enable port first */
2300         hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2301
2302         /* Set unicast mode by default. App can change to other mode in other
2303          * API func.
2304          */
2305         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2306                                         FM10K_XCAST_MODE_NONE);
2307
2308         fm10k_mbx_unlock(hw);
2309
2310         /* Add default mac address */
2311         fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2312                 MAIN_VSI_POOL_NUMBER);
2313
2314         return 0;
2315 }
2316
2317 static int
2318 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2319 {
2320         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2321
2322         PMD_INIT_FUNC_TRACE();
2323
2324         /* only uninitialize in the primary process */
2325         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2326                 return 0;
2327
2328         /* safe to close dev here */
2329         fm10k_dev_close(dev);
2330
2331         dev->dev_ops = NULL;
2332         dev->rx_pkt_burst = NULL;
2333         dev->tx_pkt_burst = NULL;
2334
2335         /* disable uio/vfio intr */
2336         rte_intr_disable(&(dev->pci_dev->intr_handle));
2337
2338         /*PF/VF has different interrupt handling mechanism */
2339         if (hw->mac.type == fm10k_mac_pf) {
2340                 /* disable interrupt */
2341                 fm10k_dev_disable_intr_pf(dev);
2342
2343                 /* unregister callback func to eal lib */
2344                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2345                         fm10k_dev_interrupt_handler_pf, (void *)dev);
2346         } else {
2347                 /* disable interrupt */
2348                 fm10k_dev_disable_intr_vf(dev);
2349
2350                 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2351                         fm10k_dev_interrupt_handler_vf, (void *)dev);
2352         }
2353
2354         /* free mac memory */
2355         if (dev->data->mac_addrs) {
2356                 rte_free(dev->data->mac_addrs);
2357                 dev->data->mac_addrs = NULL;
2358         }
2359
2360         memset(hw, 0, sizeof(*hw));
2361
2362         return 0;
2363 }
2364
2365 /*
2366  * The set of PCI devices this driver supports. This driver will enable both PF
2367  * and SRIOV-VF devices.
2368  */
2369 static const struct rte_pci_id pci_id_fm10k_map[] = {
2370 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2371 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2372 #include "rte_pci_dev_ids.h"
2373         { .vendor_id = 0, /* sentinel */ },
2374 };
2375
2376 static struct eth_driver rte_pmd_fm10k = {
2377         .pci_drv = {
2378                 .name = "rte_pmd_fm10k",
2379                 .id_table = pci_id_fm10k_map,
2380                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2381         },
2382         .eth_dev_init = eth_fm10k_dev_init,
2383         .eth_dev_uninit = eth_fm10k_dev_uninit,
2384         .dev_private_size = sizeof(struct fm10k_adapter),
2385 };
2386
2387 /*
2388  * Driver initialization routine.
2389  * Invoked once at EAL init time.
2390  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2391  */
2392 static int
2393 rte_pmd_fm10k_init(__rte_unused const char *name,
2394         __rte_unused const char *params)
2395 {
2396         PMD_INIT_FUNC_TRACE();
2397         rte_eth_driver_register(&rte_pmd_fm10k);
2398         return 0;
2399 }
2400
2401 static struct rte_driver rte_fm10k_driver = {
2402         .type = PMD_PDEV,
2403         .init = rte_pmd_fm10k_init,
2404 };
2405
2406 PMD_REGISTER_DRIVER(rte_fm10k_driver);