lib/librte_pmd_fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 #define FM10K_RX_BUFF_ALIGN 512
  45 /* Default delay to acquire mailbox lock */
  46 #define FM10K_MBXLOCK_DELAY_US 20
  47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  48
  49 /* Number of chars per uint32 type */
  50 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  51 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  52
  53 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  54
  55 static void
  56 fm10k_mbx_initlock(struct fm10k_hw *hw)
  57 {
  58         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  59 }
  60
  61 static void
  62 fm10k_mbx_lock(struct fm10k_hw *hw)
  63 {
  64         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  65                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  66 }
  67
  68 static void
  69 fm10k_mbx_unlock(struct fm10k_hw *hw)
  70 {
  71         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  72 }
  73
  74 /*
  75  * reset queue to initial state, allocate software buffers used when starting
  76  * device.
  77  * return 0 on success
  78  * return -ENOMEM if buffers cannot be allocated
  79  * return -EINVAL if buffers do not satisfy alignment condition
  80  */
  81 static inline int
  82 rx_queue_reset(struct fm10k_rx_queue *q)
  83 {
  84         uint64_t dma_addr;
  85         int i, diag;
  86         PMD_INIT_FUNC_TRACE();
  87
  88         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
  89         if (diag != 0)
  90                 return -ENOMEM;
  91
  92         for (i = 0; i < q->nb_desc; ++i) {
  93                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
  94                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
  95                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
  96                                                 q->nb_desc);
  97                         return -EINVAL;
  98                 }
  99                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 100                 q->hw_ring[i].q.pkt_addr = dma_addr;
 101                 q->hw_ring[i].q.hdr_addr = dma_addr;
 102         }
 103
 104         q->next_dd = 0;
 105         q->next_alloc = 0;
 106         q->next_trigger = q->alloc_thresh - 1;
 107         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 108         return 0;
 109 }
 110
 111 /*
 112  * clean queue, descriptor rings, free software buffers used when stopping
 113  * device.
 114  */
 115 static inline void
 116 rx_queue_clean(struct fm10k_rx_queue *q)
 117 {
 118         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 119         uint32_t i;
 120         PMD_INIT_FUNC_TRACE();
 121
 122         /* zero descriptor rings */
 123         for (i = 0; i < q->nb_desc; ++i)
 124                 q->hw_ring[i] = zero;
 125
 126         /* free software buffers */
 127         for (i = 0; i < q->nb_desc; ++i) {
 128                 if (q->sw_ring[i]) {
 129                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 130                         q->sw_ring[i] = NULL;
 131                 }
 132         }
 133 }
 134
 135 /*
 136  * free all queue memory used when releasing the queue (i.e. configure)
 137  */
 138 static inline void
 139 rx_queue_free(struct fm10k_rx_queue *q)
 140 {
 141         PMD_INIT_FUNC_TRACE();
 142         if (q) {
 143                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 144                 rx_queue_clean(q);
 145                 if (q->sw_ring) {
 146                         rte_free(q->sw_ring);
 147                         q->sw_ring = NULL;
 148                 }
 149                 rte_free(q);
 150                 q = NULL;
 151         }
 152 }
 153
 154 /*
 155  * disable RX queue, wait unitl HW finished necessary flush operation
 156  */
 157 static inline int
 158 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 159 {
 160         uint32_t reg, i;
 161
 162         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 163         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 164                         reg & ~FM10K_RXQCTL_ENABLE);
 165
 166         /* Wait 100us at most */
 167         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 168                 rte_delay_us(1);
 169                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 170                 if (!(reg & FM10K_RXQCTL_ENABLE))
 171                         break;
 172         }
 173
 174         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 175                 return -1;
 176
 177         return 0;
 178 }
 179
 180 /*
 181  * reset queue to initial state, allocate software buffers used when starting
 182  * device
 183  */
 184 static inline void
 185 tx_queue_reset(struct fm10k_tx_queue *q)
 186 {
 187         PMD_INIT_FUNC_TRACE();
 188         q->last_free = 0;
 189         q->next_free = 0;
 190         q->nb_used = 0;
 191         q->nb_free = q->nb_desc - 1;
 192         q->free_trigger = q->nb_free - q->free_thresh;
 193         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 194         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 195 }
 196
 197 /*
 198  * clean queue, descriptor rings, free software buffers used when stopping
 199  * device
 200  */
 201 static inline void
 202 tx_queue_clean(struct fm10k_tx_queue *q)
 203 {
 204         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 205         uint32_t i;
 206         PMD_INIT_FUNC_TRACE();
 207
 208         /* zero descriptor rings */
 209         for (i = 0; i < q->nb_desc; ++i)
 210                 q->hw_ring[i] = zero;
 211
 212         /* free software buffers */
 213         for (i = 0; i < q->nb_desc; ++i) {
 214                 if (q->sw_ring[i]) {
 215                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 216                         q->sw_ring[i] = NULL;
 217                 }
 218         }
 219 }
 220
 221 /*
 222  * free all queue memory used when releasing the queue (i.e. configure)
 223  */
 224 static inline void
 225 tx_queue_free(struct fm10k_tx_queue *q)
 226 {
 227         PMD_INIT_FUNC_TRACE();
 228         if (q) {
 229                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 230                 tx_queue_clean(q);
 231                 if (q->rs_tracker.list) {
 232                         rte_free(q->rs_tracker.list);
 233                         q->rs_tracker.list = NULL;
 234                 }
 235                 if (q->sw_ring) {
 236                         rte_free(q->sw_ring);
 237                         q->sw_ring = NULL;
 238                 }
 239                 rte_free(q);
 240                 q = NULL;
 241         }
 242 }
 243
 244 /*
 245  * disable TX queue, wait unitl HW finished necessary flush operation
 246  */
 247 static inline int
 248 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 249 {
 250         uint32_t reg, i;
 251
 252         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 253         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 254                         reg & ~FM10K_TXDCTL_ENABLE);
 255
 256         /* Wait 100us at most */
 257         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 258                 rte_delay_us(1);
 259                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 260                 if (!(reg & FM10K_TXDCTL_ENABLE))
 261                         break;
 262         }
 263
 264         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 265                 return -1;
 266
 267         return 0;
 268 }
 269
 270 static int
 271 fm10k_dev_configure(struct rte_eth_dev *dev)
 272 {
 273         PMD_INIT_FUNC_TRACE();
 274
 275         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 276                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 277
 278         return 0;
 279 }
 280
 281 static void
 282 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 283 {
 284         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 285         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 286         uint32_t mrqc, *key, i, reta, j;
 287         uint64_t hf;
 288
 289 #define RSS_KEY_SIZE 40
 290         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 291                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 292                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 293                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 294                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 295                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 296         };
 297
 298         if (dev->data->nb_rx_queues == 1 ||
 299             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 300             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 301                 return;
 302
 303         /* random key is rss_intel_key (default) or user provided (rss_key) */
 304         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 305                 key = (uint32_t *)rss_intel_key;
 306         else
 307                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 308
 309         /* Now fill our hash function seeds, 4 bytes at a time */
 310         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 311                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 312
 313         /*
 314          * Fill in redirection table
 315          * The byte-swap is needed because NIC registers are in
 316          * little-endian order.
 317          */
 318         reta = 0;
 319         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 320                 if (j == dev->data->nb_rx_queues)
 321                         j = 0;
 322                 reta = (reta << CHAR_BIT) | j;
 323                 if ((i & 3) == 3)
 324                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 325                                         rte_bswap32(reta));
 326         }
 327
 328         /*
 329          * Generate RSS hash based on packet types, TCP/UDP
 330          * port numbers and/or IPv4/v6 src and dst addresses
 331          */
 332         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 333         mrqc = 0;
 334         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 335         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 336         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 337         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 338         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 339         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 340         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 341         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 342         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 343
 344         if (mrqc == 0) {
 345                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 346                         "supported", hf);
 347                 return;
 348         }
 349
 350         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 351 }
 352
 353 static int
 354 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 355 {
 356         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 357         int i, ret;
 358         struct fm10k_tx_queue *txq;
 359         uint64_t base_addr;
 360         uint32_t size;
 361
 362         /* Disable TXINT to avoid possible interrupt */
 363         for (i = 0; i < hw->mac.max_queues; i++)
 364                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 365                                 3 << FM10K_TXINT_TIMER_SHIFT);
 366
 367         /* Setup TX queue */
 368         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 369                 txq = dev->data->tx_queues[i];
 370                 base_addr = txq->hw_ring_phys_addr;
 371                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 372
 373                 /* disable queue to avoid issues while updating state */
 374                 ret = tx_queue_disable(hw, i);
 375                 if (ret) {
 376                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 377                         return -1;
 378                 }
 379
 380                 /* set location and size for descriptor ring */
 381                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 382                                 base_addr & UINT64_LOWER_32BITS_MASK);
 383                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 384                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 385                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 386         }
 387         return 0;
 388 }
 389
 390 static int
 391 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 392 {
 393         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 394         int i, ret;
 395         struct fm10k_rx_queue *rxq;
 396         uint64_t base_addr;
 397         uint32_t size;
 398         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 399         uint16_t buf_size;
 400         struct rte_pktmbuf_pool_private *mbp_priv;
 401
 402         /* Disable RXINT to avoid possible interrupt */
 403         for (i = 0; i < hw->mac.max_queues; i++)
 404                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 405                                 3 << FM10K_RXINT_TIMER_SHIFT);
 406
 407         /* Setup RX queues */
 408         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 409                 rxq = dev->data->rx_queues[i];
 410                 base_addr = rxq->hw_ring_phys_addr;
 411                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 412
 413                 /* disable queue to avoid issues while updating state */
 414                 ret = rx_queue_disable(hw, i);
 415                 if (ret) {
 416                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 417                         return -1;
 418                 }
 419
 420                 /* Setup the Base and Length of the Rx Descriptor Ring */
 421                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 422                                 base_addr & UINT64_LOWER_32BITS_MASK);
 423                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 424                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 425                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 426
 427                 /* Configure the Rx buffer size for one buff without split */
 428                 mbp_priv = rte_mempool_get_priv(rxq->mp);
 429                 buf_size = (uint16_t) (mbp_priv->mbuf_data_room_size -
 430                                         RTE_PKTMBUF_HEADROOM);
 431                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 432                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 433
 434                 /* It adds dual VLAN length for supporting dual VLAN */
 435                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 436                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size){
 437                         dev->data->scattered_rx = 1;
 438                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 439                 }
 440
 441                 /* Enable drop on empty, it's RO for VF */
 442                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 443                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 444
 445                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 446                 FM10K_WRITE_FLUSH(hw);
 447         }
 448
 449         if (dev->data->dev_conf.rxmode.enable_scatter) {
 450                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 451                 dev->data->scattered_rx = 1;
 452         }
 453
 454         /* Configure RSS if applicable */
 455         fm10k_dev_mq_rx_configure(dev);
 456         return 0;
 457 }
 458
 459 static int
 460 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 461 {
 462         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 463         int err = -1;
 464         uint32_t reg;
 465         struct fm10k_rx_queue *rxq;
 466
 467         PMD_INIT_FUNC_TRACE();
 468
 469         if (rx_queue_id < dev->data->nb_rx_queues) {
 470                 rxq = dev->data->rx_queues[rx_queue_id];
 471                 err = rx_queue_reset(rxq);
 472                 if (err == -ENOMEM) {
 473                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 474                         return err;
 475                 } else if (err == -EINVAL) {
 476                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 477                                 " %d", err);
 478                         return err;
 479                 }
 480
 481                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 482                  * Note: this must be done AFTER the queue is enabled on real
 483                  * hardware, but BEFORE the queue is enabled when using the
 484                  * emulation platform. Do it in both places for now and remove
 485                  * this comment and the following two register writes when the
 486                  * emulation platform is no longer being used.
 487                  */
 488                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 489                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 490
 491                 /* Set PF ownership flag for PF devices */
 492                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 493                 if (hw->mac.type == fm10k_mac_pf)
 494                         reg |= FM10K_RXQCTL_PF;
 495                 reg |= FM10K_RXQCTL_ENABLE;
 496                 /* enable RX queue */
 497                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 498                 FM10K_WRITE_FLUSH(hw);
 499
 500                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 501                  * Note: this must be done AFTER the queue is enabled
 502                  */
 503                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 504                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 505         }
 506
 507         return err;
 508 }
 509
 510 static int
 511 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 512 {
 513         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 514
 515         PMD_INIT_FUNC_TRACE();
 516
 517         if (rx_queue_id < dev->data->nb_rx_queues) {
 518                 /* Disable RX queue */
 519                 rx_queue_disable(hw, rx_queue_id);
 520
 521                 /* Free mbuf and clean HW ring */
 522                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 523         }
 524
 525         return 0;
 526 }
 527
 528 static int
 529 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 530 {
 531         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 532         /** @todo - this should be defined in the shared code */
 533 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 534         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 535         int err = 0;
 536
 537         PMD_INIT_FUNC_TRACE();
 538
 539         if (tx_queue_id < dev->data->nb_tx_queues) {
 540                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 541
 542                 /* reset head and tail pointers */
 543                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 544                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 545
 546                 /* enable TX queue */
 547                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 548                                         FM10K_TXDCTL_ENABLE | txdctl);
 549                 FM10K_WRITE_FLUSH(hw);
 550         } else
 551                 err = -1;
 552
 553         return err;
 554 }
 555
 556 static int
 557 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 558 {
 559         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 560
 561         PMD_INIT_FUNC_TRACE();
 562
 563         if (tx_queue_id < dev->data->nb_tx_queues) {
 564                 tx_queue_disable(hw, tx_queue_id);
 565                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 566         }
 567
 568         return 0;
 569 }
 570
 571 /* fls = find last set bit = 32 minus the number of leading zeros */
 572 #ifndef fls
 573 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 574 #endif
 575 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 576 static int
 577 fm10k_dev_start(struct rte_eth_dev *dev)
 578 {
 579         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 580         int i, diag;
 581
 582         PMD_INIT_FUNC_TRACE();
 583
 584         /* stop, init, then start the hw */
 585         diag = fm10k_stop_hw(hw);
 586         if (diag != FM10K_SUCCESS) {
 587                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 588                 return -EIO;
 589         }
 590
 591         diag = fm10k_init_hw(hw);
 592         if (diag != FM10K_SUCCESS) {
 593                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 594                 return -EIO;
 595         }
 596
 597         diag = fm10k_start_hw(hw);
 598         if (diag != FM10K_SUCCESS) {
 599                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 600                 return -EIO;
 601         }
 602
 603         diag = fm10k_dev_tx_init(dev);
 604         if (diag) {
 605                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 606                 return diag;
 607         }
 608
 609         diag = fm10k_dev_rx_init(dev);
 610         if (diag) {
 611                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 612                 return diag;
 613         }
 614
 615         if (hw->mac.type == fm10k_mac_pf) {
 616                 /* Establish only VSI 0 as valid */
 617                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 618
 619                 /* Configure RSS bits used in RETA table */
 620                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 621                                 fls(dev->data->nb_rx_queues - 1) <<
 622                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 623
 624                 /* Invalidate all other GLORT entries */
 625                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 626                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 627                                         FM10K_DGLORTMAP_NONE);
 628         }
 629
 630         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 631                 struct fm10k_rx_queue *rxq;
 632                 rxq = dev->data->rx_queues[i];
 633
 634                 if (rxq->rx_deferred_start)
 635                         continue;
 636                 diag = fm10k_dev_rx_queue_start(dev, i);
 637                 if (diag != 0) {
 638                         int j;
 639                         for (j = 0; j < i; ++j)
 640                                 rx_queue_clean(dev->data->rx_queues[j]);
 641                         return diag;
 642                 }
 643         }
 644
 645         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 646                 struct fm10k_tx_queue *txq;
 647                 txq = dev->data->tx_queues[i];
 648
 649                 if (txq->tx_deferred_start)
 650                         continue;
 651                 diag = fm10k_dev_tx_queue_start(dev, i);
 652                 if (diag != 0) {
 653                         int j;
 654                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 655                                 rx_queue_clean(dev->data->rx_queues[j]);
 656                         return diag;
 657                 }
 658         }
 659
 660         return 0;
 661 }
 662
 663 static void
 664 fm10k_dev_stop(struct rte_eth_dev *dev)
 665 {
 666         int i;
 667
 668         PMD_INIT_FUNC_TRACE();
 669
 670         for (i = 0; i < dev->data->nb_tx_queues; i++)
 671                 fm10k_dev_tx_queue_stop(dev, i);
 672
 673         for (i = 0; i < dev->data->nb_rx_queues; i++)
 674                 fm10k_dev_rx_queue_stop(dev, i);
 675 }
 676
 677 static void
 678 fm10k_dev_close(struct rte_eth_dev *dev)
 679 {
 680         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 681
 682         PMD_INIT_FUNC_TRACE();
 683
 684         /* Stop mailbox service first */
 685         fm10k_close_mbx_service(hw);
 686         fm10k_dev_stop(dev);
 687         fm10k_stop_hw(hw);
 688 }
 689
 690 static int
 691 fm10k_link_update(struct rte_eth_dev *dev,
 692         __rte_unused int wait_to_complete)
 693 {
 694         PMD_INIT_FUNC_TRACE();
 695
 696         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 697          * x8 PCIe interface. For now, we leave the speed undefined since there
 698          * is no 50Gbps Ethernet. */
 699         dev->data->dev_link.link_speed  = 0;
 700         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 701         dev->data->dev_link.link_status = 1;
 702
 703         return 0;
 704 }
 705
 706 static void
 707 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 708 {
 709         uint64_t ipackets, opackets, ibytes, obytes;
 710         struct fm10k_hw *hw =
 711                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 712         struct fm10k_hw_stats *hw_stats =
 713                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 714         int i;
 715
 716         PMD_INIT_FUNC_TRACE();
 717
 718         fm10k_update_hw_stats(hw, hw_stats);
 719
 720         ipackets = opackets = ibytes = obytes = 0;
 721         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 722                 (i < FM10K_MAX_QUEUES_PF); ++i) {
 723                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 724                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 725                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 726                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 727                 ipackets += stats->q_ipackets[i];
 728                 opackets += stats->q_opackets[i];
 729                 ibytes   += stats->q_ibytes[i];
 730                 obytes   += stats->q_obytes[i];
 731         }
 732         stats->ipackets = ipackets;
 733         stats->opackets = opackets;
 734         stats->ibytes = ibytes;
 735         stats->obytes = obytes;
 736 }
 737
 738 static void
 739 fm10k_stats_reset(struct rte_eth_dev *dev)
 740 {
 741         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 742         struct fm10k_hw_stats *hw_stats =
 743                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 744
 745         PMD_INIT_FUNC_TRACE();
 746
 747         memset(hw_stats, 0, sizeof(*hw_stats));
 748         fm10k_rebind_hw_stats(hw, hw_stats);
 749 }
 750
 751 static void
 752 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 753         struct rte_eth_dev_info *dev_info)
 754 {
 755         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 756
 757         PMD_INIT_FUNC_TRACE();
 758
 759         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 760         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 761         dev_info->max_rx_queues      = hw->mac.max_queues;
 762         dev_info->max_tx_queues      = hw->mac.max_queues;
 763         dev_info->max_mac_addrs      = 1;
 764         dev_info->max_hash_mac_addrs = 0;
 765         dev_info->max_vfs            = FM10K_MAX_VF_NUM;
 766         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 767         dev_info->rx_offload_capa =
 768                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 769                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 770                 DEV_RX_OFFLOAD_TCP_CKSUM;
 771         dev_info->tx_offload_capa    = 0;
 772         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 773
 774         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 775                 .rx_thresh = {
 776                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 777                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 778                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 779                 },
 780                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 781                 .rx_drop_en = 0,
 782         };
 783
 784         dev_info->default_txconf = (struct rte_eth_txconf) {
 785                 .tx_thresh = {
 786                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 787                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 788                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 789                 },
 790                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 791                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 792                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 793                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 794         };
 795
 796 }
 797
 798 static int
 799 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 800 {
 801         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 802
 803         PMD_INIT_FUNC_TRACE();
 804
 805         /* @todo - add support for the VF */
 806         if (hw->mac.type != fm10k_mac_pf)
 807                 return -ENOTSUP;
 808
 809         return fm10k_update_vlan(hw, vlan_id, 0, on);
 810 }
 811
 812 static inline int
 813 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
 814 {
 815         if ((request < min) || (request > max) || ((request % mult) != 0))
 816                 return -1;
 817         else
 818                 return 0;
 819 }
 820
 821 /*
 822  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
 823  * the physical address is required. If the memzone is already created, then
 824  * this function returns a pointer to the existing memzone.
 825  */
 826 static inline const struct rte_memzone *
 827 allocate_hw_ring(const char *driver_name, const char *ring_name,
 828         uint8_t port_id, uint16_t queue_id, int socket_id,
 829         uint32_t size, uint32_t align)
 830 {
 831         char name[RTE_MEMZONE_NAMESIZE];
 832         const struct rte_memzone *mz;
 833
 834         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
 835                  driver_name, ring_name, port_id, queue_id, socket_id);
 836
 837         /* return the memzone if it already exists */
 838         mz = rte_memzone_lookup(name);
 839         if (mz)
 840                 return mz;
 841
 842 #ifdef RTE_LIBRTE_XEN_DOM0
 843         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
 844                                            RTE_PGSIZE_2M);
 845 #else
 846         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
 847 #endif
 848 }
 849
 850 static inline int
 851 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
 852 {
 853         if ((request < min) || (request > max) || ((div % request) != 0))
 854                 return -1;
 855         else
 856                 return 0;
 857 }
 858
 859 static inline int
 860 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
 861 {
 862         uint16_t rx_free_thresh;
 863
 864         if (conf->rx_free_thresh == 0)
 865                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
 866         else
 867                 rx_free_thresh = conf->rx_free_thresh;
 868
 869         /* make sure the requested threshold satisfies the constraints */
 870         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
 871                         FM10K_RX_FREE_THRESH_MAX(q),
 872                         FM10K_RX_FREE_THRESH_DIV(q),
 873                         rx_free_thresh)) {
 874                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
 875                         "less than or equal to %u, "
 876                         "greater than or equal to %u, "
 877                         "and a divisor of %u",
 878                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
 879                         FM10K_RX_FREE_THRESH_MIN(q),
 880                         FM10K_RX_FREE_THRESH_DIV(q));
 881                 return (-EINVAL);
 882         }
 883
 884         q->alloc_thresh = rx_free_thresh;
 885         q->drop_en = conf->rx_drop_en;
 886         q->rx_deferred_start = conf->rx_deferred_start;
 887
 888         return 0;
 889 }
 890
 891 /*
 892  * Hardware requires specific alignment for Rx packet buffers. At
 893  * least one of the following two conditions must be satisfied.
 894  *  1. Address is 512B aligned
 895  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
 896  *
 897  * As such, the driver may need to adjust the DMA address within the
 898  * buffer by up to 512B. The mempool element size is checked here
 899  * to make sure a maximally sized Ethernet frame can still be wholly
 900  * contained within the buffer after 512B alignment.
 901  *
 902  * return 1 if the element size is valid, otherwise return 0.
 903  */
 904 static int
 905 mempool_element_size_valid(struct rte_mempool *mp)
 906 {
 907         uint32_t min_size;
 908
 909         /* elt_size includes mbuf header and headroom */
 910         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
 911                         RTE_PKTMBUF_HEADROOM;
 912
 913         /* account for up to 512B of alignment */
 914         min_size -= FM10K_RX_BUFF_ALIGN;
 915
 916         /* sanity check for overflow */
 917         if (min_size > mp->elt_size)
 918                 return 0;
 919
 920         if (min_size < ETHER_MAX_VLAN_FRAME_LEN)
 921                 return 0;
 922
 923         /* size is valid */
 924         return 1;
 925 }
 926
 927 static int
 928 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 929         uint16_t nb_desc, unsigned int socket_id,
 930         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
 931 {
 932         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 933         struct fm10k_rx_queue *q;
 934         const struct rte_memzone *mz;
 935
 936         PMD_INIT_FUNC_TRACE();
 937
 938         /* make sure the mempool element size can account for alignment. */
 939         if (!mempool_element_size_valid(mp)) {
 940                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
 941                 return (-EINVAL);
 942         }
 943
 944         /* make sure a valid number of descriptors have been requested */
 945         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
 946                                 FM10K_MULT_RX_DESC, nb_desc)) {
 947                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
 948                         "less than or equal to %"PRIu32", "
 949                         "greater than or equal to %u, "
 950                         "and a multiple of %u",
 951                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
 952                         FM10K_MULT_RX_DESC);
 953                 return (-EINVAL);
 954         }
 955
 956         /*
 957          * if this queue existed already, free the associated memory. The
 958          * queue cannot be reused in case we need to allocate memory on
 959          * different socket than was previously used.
 960          */
 961         if (dev->data->rx_queues[queue_id] != NULL) {
 962                 rx_queue_free(dev->data->rx_queues[queue_id]);
 963                 dev->data->rx_queues[queue_id] = NULL;
 964         }
 965
 966         /* allocate memory for the queue structure */
 967         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
 968                                 socket_id);
 969         if (q == NULL) {
 970                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
 971                 return (-ENOMEM);
 972         }
 973
 974         /* setup queue */
 975         q->mp = mp;
 976         q->nb_desc = nb_desc;
 977         q->port_id = dev->data->port_id;
 978         q->queue_id = queue_id;
 979         q->tail_ptr = (volatile uint32_t *)
 980                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
 981         if (handle_rxconf(q, conf))
 982                 return (-EINVAL);
 983
 984         /* allocate memory for the software ring */
 985         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
 986                                         nb_desc * sizeof(struct rte_mbuf *),
 987                                         RTE_CACHE_LINE_SIZE, socket_id);
 988         if (q->sw_ring == NULL) {
 989                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
 990                 rte_free(q);
 991                 return (-ENOMEM);
 992         }
 993
 994         /*
 995          * allocate memory for the hardware descriptor ring. A memzone large
 996          * enough to hold the maximum ring size is requested to allow for
 997          * resizing in later calls to the queue setup function.
 998          */
 999         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1000                                 dev->data->port_id, queue_id, socket_id,
1001                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1002         if (mz == NULL) {
1003                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1004                 rte_free(q->sw_ring);
1005                 rte_free(q);
1006                 return (-ENOMEM);
1007         }
1008         q->hw_ring = mz->addr;
1009         q->hw_ring_phys_addr = mz->phys_addr;
1010
1011         dev->data->rx_queues[queue_id] = q;
1012         return 0;
1013 }
1014
1015 static void
1016 fm10k_rx_queue_release(void *queue)
1017 {
1018         PMD_INIT_FUNC_TRACE();
1019
1020         rx_queue_free(queue);
1021 }
1022
1023 static inline int
1024 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1025 {
1026         uint16_t tx_free_thresh;
1027         uint16_t tx_rs_thresh;
1028
1029         /* constraint MACROs require that tx_free_thresh is configured
1030          * before tx_rs_thresh */
1031         if (conf->tx_free_thresh == 0)
1032                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1033         else
1034                 tx_free_thresh = conf->tx_free_thresh;
1035
1036         /* make sure the requested threshold satisfies the constraints */
1037         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1038                         FM10K_TX_FREE_THRESH_MAX(q),
1039                         FM10K_TX_FREE_THRESH_DIV(q),
1040                         tx_free_thresh)) {
1041                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1042                         "less than or equal to %u, "
1043                         "greater than or equal to %u, "
1044                         "and a divisor of %u",
1045                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1046                         FM10K_TX_FREE_THRESH_MIN(q),
1047                         FM10K_TX_FREE_THRESH_DIV(q));
1048                 return (-EINVAL);
1049         }
1050
1051         q->free_thresh = tx_free_thresh;
1052
1053         if (conf->tx_rs_thresh == 0)
1054                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1055         else
1056                 tx_rs_thresh = conf->tx_rs_thresh;
1057
1058         q->tx_deferred_start = conf->tx_deferred_start;
1059
1060         /* make sure the requested threshold satisfies the constraints */
1061         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1062                         FM10K_TX_RS_THRESH_MAX(q),
1063                         FM10K_TX_RS_THRESH_DIV(q),
1064                         tx_rs_thresh)) {
1065                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1066                         "less than or equal to %u, "
1067                         "greater than or equal to %u, "
1068                         "and a divisor of %u",
1069                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1070                         FM10K_TX_RS_THRESH_MIN(q),
1071                         FM10K_TX_RS_THRESH_DIV(q));
1072                 return (-EINVAL);
1073         }
1074
1075         q->rs_thresh = tx_rs_thresh;
1076
1077         return 0;
1078 }
1079
1080 static int
1081 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1082         uint16_t nb_desc, unsigned int socket_id,
1083         const struct rte_eth_txconf *conf)
1084 {
1085         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1086         struct fm10k_tx_queue *q;
1087         const struct rte_memzone *mz;
1088
1089         PMD_INIT_FUNC_TRACE();
1090
1091         /* make sure a valid number of descriptors have been requested */
1092         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1093                                 FM10K_MULT_TX_DESC, nb_desc)) {
1094                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1095                         "less than or equal to %"PRIu32", "
1096                         "greater than or equal to %u, "
1097                         "and a multiple of %u",
1098                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1099                         FM10K_MULT_TX_DESC);
1100                 return (-EINVAL);
1101         }
1102
1103         /*
1104          * if this queue existed already, free the associated memory. The
1105          * queue cannot be reused in case we need to allocate memory on
1106          * different socket than was previously used.
1107          */
1108         if (dev->data->tx_queues[queue_id] != NULL) {
1109                 tx_queue_free(dev->data->tx_queues[queue_id]);
1110                 dev->data->tx_queues[queue_id] = NULL;
1111         }
1112
1113         /* allocate memory for the queue structure */
1114         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1115                                 socket_id);
1116         if (q == NULL) {
1117                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1118                 return (-ENOMEM);
1119         }
1120
1121         /* setup queue */
1122         q->nb_desc = nb_desc;
1123         q->port_id = dev->data->port_id;
1124         q->queue_id = queue_id;
1125         q->tail_ptr = (volatile uint32_t *)
1126                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1127         if (handle_txconf(q, conf))
1128                 return (-EINVAL);
1129
1130         /* allocate memory for the software ring */
1131         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1132                                         nb_desc * sizeof(struct rte_mbuf *),
1133                                         RTE_CACHE_LINE_SIZE, socket_id);
1134         if (q->sw_ring == NULL) {
1135                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1136                 rte_free(q);
1137                 return (-ENOMEM);
1138         }
1139
1140         /*
1141          * allocate memory for the hardware descriptor ring. A memzone large
1142          * enough to hold the maximum ring size is requested to allow for
1143          * resizing in later calls to the queue setup function.
1144          */
1145         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1146                                 dev->data->port_id, queue_id, socket_id,
1147                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1148         if (mz == NULL) {
1149                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1150                 rte_free(q->sw_ring);
1151                 rte_free(q);
1152                 return (-ENOMEM);
1153         }
1154         q->hw_ring = mz->addr;
1155         q->hw_ring_phys_addr = mz->phys_addr;
1156
1157         /*
1158          * allocate memory for the RS bit tracker. Enough slots to hold the
1159          * descriptor index for each RS bit needing to be set are required.
1160          */
1161         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1162                                 ((nb_desc + 1) / q->rs_thresh) *
1163                                 sizeof(uint16_t),
1164                                 RTE_CACHE_LINE_SIZE, socket_id);
1165         if (q->rs_tracker.list == NULL) {
1166                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1167                 rte_free(q->sw_ring);
1168                 rte_free(q);
1169                 return (-ENOMEM);
1170         }
1171
1172         dev->data->tx_queues[queue_id] = q;
1173         return 0;
1174 }
1175
1176 static void
1177 fm10k_tx_queue_release(void *queue)
1178 {
1179         PMD_INIT_FUNC_TRACE();
1180
1181         tx_queue_free(queue);
1182 }
1183
1184 static int
1185 fm10k_reta_update(struct rte_eth_dev *dev,
1186                         struct rte_eth_rss_reta_entry64 *reta_conf,
1187                         uint16_t reta_size)
1188 {
1189         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1190         uint16_t i, j, idx, shift;
1191         uint8_t mask;
1192         uint32_t reta;
1193
1194         PMD_INIT_FUNC_TRACE();
1195
1196         if (reta_size > FM10K_MAX_RSS_INDICES) {
1197                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1198                         "(%d) doesn't match the number hardware can supported "
1199                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1200                 return -EINVAL;
1201         }
1202
1203         /*
1204          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1205          * 128-entries in 32 registers
1206          */
1207         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1208                 idx = i / RTE_RETA_GROUP_SIZE;
1209                 shift = i % RTE_RETA_GROUP_SIZE;
1210                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1211                                 BIT_MASK_PER_UINT32);
1212                 if (mask == 0)
1213                         continue;
1214
1215                 reta = 0;
1216                 if (mask != BIT_MASK_PER_UINT32)
1217                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1218
1219                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1220                         if (mask & (0x1 << j)) {
1221                                 if (mask != 0xF)
1222                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1223                                 reta |= reta_conf[idx].reta[shift + j] <<
1224                                                 (CHAR_BIT * j);
1225                         }
1226                 }
1227                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1228         }
1229
1230         return 0;
1231 }
1232
1233 static int
1234 fm10k_reta_query(struct rte_eth_dev *dev,
1235                         struct rte_eth_rss_reta_entry64 *reta_conf,
1236                         uint16_t reta_size)
1237 {
1238         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1239         uint16_t i, j, idx, shift;
1240         uint8_t mask;
1241         uint32_t reta;
1242
1243         PMD_INIT_FUNC_TRACE();
1244
1245         if (reta_size < FM10K_MAX_RSS_INDICES) {
1246                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1247                         "(%d) doesn't match the number hardware can supported "
1248                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1249                 return -EINVAL;
1250         }
1251
1252         /*
1253          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1254          * 128-entries in 32 registers
1255          */
1256         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1257                 idx = i / RTE_RETA_GROUP_SIZE;
1258                 shift = i % RTE_RETA_GROUP_SIZE;
1259                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1260                                 BIT_MASK_PER_UINT32);
1261                 if (mask == 0)
1262                         continue;
1263
1264                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1265                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1266                         if (mask & (0x1 << j))
1267                                 reta_conf[idx].reta[shift + j] = ((reta >>
1268                                         CHAR_BIT * j) & UINT8_MAX);
1269                 }
1270         }
1271
1272         return 0;
1273 }
1274
1275 static int
1276 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1277         struct rte_eth_rss_conf *rss_conf)
1278 {
1279         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1280         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1281         uint32_t mrqc;
1282         uint64_t hf = rss_conf->rss_hf;
1283         int i;
1284
1285         PMD_INIT_FUNC_TRACE();
1286
1287         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1288                 FM10K_RSSRK_ENTRIES_PER_REG)
1289                 return -EINVAL;
1290
1291         if (hf == 0)
1292                 return -EINVAL;
1293
1294         mrqc = 0;
1295         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1296         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1297         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1298         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1299         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1300         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1301         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1302         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1303         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1304
1305         /* If the mapping doesn't fit any supported, return */
1306         if (mrqc == 0)
1307                 return -EINVAL;
1308
1309         if (key != NULL)
1310                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1311                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1312
1313         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1314
1315         return 0;
1316 }
1317
1318 static int
1319 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1320         struct rte_eth_rss_conf *rss_conf)
1321 {
1322         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1323         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1324         uint32_t mrqc;
1325         uint64_t hf;
1326         int i;
1327
1328         PMD_INIT_FUNC_TRACE();
1329
1330         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1331                                 FM10K_RSSRK_ENTRIES_PER_REG)
1332                 return -EINVAL;
1333
1334         if (key != NULL)
1335                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1336                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1337
1338         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1339         hf = 0;
1340         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1341         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1342         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1343         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1344         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1345         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1346         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1347         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1348         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1349
1350         rss_conf->rss_hf = hf;
1351
1352         return 0;
1353 }
1354
1355 static void
1356 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1357 {
1358         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1359         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1360
1361         /* Bind all local non-queue interrupt to vector 0 */
1362         int_map |= 0;
1363
1364         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1365         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1366         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1367         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1368         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1369         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1370
1371         /* Enable misc causes */
1372         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1373                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1374                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1375                                 FM10K_EIMR_ENABLE(MAILBOX) |
1376                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1377                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1378                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1379                                 FM10K_EIMR_ENABLE(VFLR));
1380
1381         /* Enable ITR 0 */
1382         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1383                                         FM10K_ITR_MASK_CLEAR);
1384         FM10K_WRITE_FLUSH(hw);
1385 }
1386
1387 static void
1388 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1389 {
1390         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1391         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1392
1393         /* Bind all local non-queue interrupt to vector 0 */
1394         int_map |= 0;
1395
1396         /* Only INT 0 available, other 15 are reserved. */
1397         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1398
1399         /* Enable ITR 0 */
1400         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1401                                         FM10K_ITR_MASK_CLEAR);
1402         FM10K_WRITE_FLUSH(hw);
1403 }
1404
1405 static int
1406 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1407 {
1408         struct fm10k_fault fault;
1409         int err;
1410         const char *estr = "Unknown error";
1411
1412         /* Process PCA fault */
1413         if (eicr & FM10K_EIMR_PCA_FAULT) {
1414                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1415                 if (err)
1416                         goto error;
1417                 switch (fault.type) {
1418                 case PCA_NO_FAULT:
1419                         estr = "PCA_NO_FAULT"; break;
1420                 case PCA_UNMAPPED_ADDR:
1421                         estr = "PCA_UNMAPPED_ADDR"; break;
1422                 case PCA_BAD_QACCESS_PF:
1423                         estr = "PCA_BAD_QACCESS_PF"; break;
1424                 case PCA_BAD_QACCESS_VF:
1425                         estr = "PCA_BAD_QACCESS_VF"; break;
1426                 case PCA_MALICIOUS_REQ:
1427                         estr = "PCA_MALICIOUS_REQ"; break;
1428                 case PCA_POISONED_TLP:
1429                         estr = "PCA_POISONED_TLP"; break;
1430                 case PCA_TLP_ABORT:
1431                         estr = "PCA_TLP_ABORT"; break;
1432                 default:
1433                         goto error;
1434                 }
1435                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1436                         estr, fault.func ? "VF" : "PF", fault.func,
1437                         fault.address, fault.specinfo);
1438         }
1439
1440         /* Process THI fault */
1441         if (eicr & FM10K_EIMR_THI_FAULT) {
1442                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1443                 if (err)
1444                         goto error;
1445                 switch (fault.type) {
1446                 case THI_NO_FAULT:
1447                         estr = "THI_NO_FAULT"; break;
1448                 case THI_MAL_DIS_Q_FAULT:
1449                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1450                 default:
1451                         goto error;
1452                 }
1453                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1454                         estr, fault.func ? "VF" : "PF", fault.func,
1455                         fault.address, fault.specinfo);
1456         }
1457
1458         /* Process FUM fault */
1459         if (eicr & FM10K_EIMR_FUM_FAULT) {
1460                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1461                 if (err)
1462                         goto error;
1463                 switch (fault.type) {
1464                 case FUM_NO_FAULT:
1465                         estr = "FUM_NO_FAULT"; break;
1466                 case FUM_UNMAPPED_ADDR:
1467                         estr = "FUM_UNMAPPED_ADDR"; break;
1468                 case FUM_POISONED_TLP:
1469                         estr = "FUM_POISONED_TLP"; break;
1470                 case FUM_BAD_VF_QACCESS:
1471                         estr = "FUM_BAD_VF_QACCESS"; break;
1472                 case FUM_ADD_DECODE_ERR:
1473                         estr = "FUM_ADD_DECODE_ERR"; break;
1474                 case FUM_RO_ERROR:
1475                         estr = "FUM_RO_ERROR"; break;
1476                 case FUM_QPRC_CRC_ERROR:
1477                         estr = "FUM_QPRC_CRC_ERROR"; break;
1478                 case FUM_CSR_TIMEOUT:
1479                         estr = "FUM_CSR_TIMEOUT"; break;
1480                 case FUM_INVALID_TYPE:
1481                         estr = "FUM_INVALID_TYPE"; break;
1482                 case FUM_INVALID_LENGTH:
1483                         estr = "FUM_INVALID_LENGTH"; break;
1484                 case FUM_INVALID_BE:
1485                         estr = "FUM_INVALID_BE"; break;
1486                 case FUM_INVALID_ALIGN:
1487                         estr = "FUM_INVALID_ALIGN"; break;
1488                 default:
1489                         goto error;
1490                 }
1491                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1492                         estr, fault.func ? "VF" : "PF", fault.func,
1493                         fault.address, fault.specinfo);
1494         }
1495
1496         if (estr)
1497                 return 0;
1498         return 0;
1499 error:
1500         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1501         return err;
1502 }
1503
1504 /**
1505  * PF interrupt handler triggered by NIC for handling specific interrupt.
1506  *
1507  * @param handle
1508  *  Pointer to interrupt handle.
1509  * @param param
1510  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1511  *
1512  * @return
1513  *  void
1514  */
1515 static void
1516 fm10k_dev_interrupt_handler_pf(
1517                         __rte_unused struct rte_intr_handle *handle,
1518                         void *param)
1519 {
1520         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1521         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1522         uint32_t cause, status;
1523
1524         if (hw->mac.type != fm10k_mac_pf)
1525                 return;
1526
1527         cause = FM10K_READ_REG(hw, FM10K_EICR);
1528
1529         /* Handle PCI fault cases */
1530         if (cause & FM10K_EICR_FAULT_MASK) {
1531                 PMD_INIT_LOG(ERR, "INT: find fault!");
1532                 fm10k_dev_handle_fault(hw, cause);
1533         }
1534
1535         /* Handle switch up/down */
1536         if (cause & FM10K_EICR_SWITCHNOTREADY)
1537                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1538
1539         if (cause & FM10K_EICR_SWITCHREADY)
1540                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1541
1542         /* Handle mailbox message */
1543         fm10k_mbx_lock(hw);
1544         hw->mbx.ops.process(hw, &hw->mbx);
1545         fm10k_mbx_unlock(hw);
1546
1547         /* Handle SRAM error */
1548         if (cause & FM10K_EICR_SRAMERROR) {
1549                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1550
1551                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1552                 /* Write to clear pending bits */
1553                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1554
1555                 /* Todo: print out error message after shared code  updates */
1556         }
1557
1558         /* Clear these 3 events if having any */
1559         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1560                  FM10K_EICR_SWITCHREADY;
1561         if (cause)
1562                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1563
1564         /* Re-enable interrupt from device side */
1565         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1566                                         FM10K_ITR_MASK_CLEAR);
1567         /* Re-enable interrupt from host side */
1568         rte_intr_enable(&(dev->pci_dev->intr_handle));
1569 }
1570
1571 /**
1572  * VF interrupt handler triggered by NIC for handling specific interrupt.
1573  *
1574  * @param handle
1575  *  Pointer to interrupt handle.
1576  * @param param
1577  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1578  *
1579  * @return
1580  *  void
1581  */
1582 static void
1583 fm10k_dev_interrupt_handler_vf(
1584                         __rte_unused struct rte_intr_handle *handle,
1585                         void *param)
1586 {
1587         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1588         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1589
1590         if (hw->mac.type != fm10k_mac_vf)
1591                 return;
1592
1593         /* Handle mailbox message if lock is acquired */
1594         fm10k_mbx_lock(hw);
1595         hw->mbx.ops.process(hw, &hw->mbx);
1596         fm10k_mbx_unlock(hw);
1597
1598         /* Re-enable interrupt from device side */
1599         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1600                                         FM10K_ITR_MASK_CLEAR);
1601         /* Re-enable interrupt from host side */
1602         rte_intr_enable(&(dev->pci_dev->intr_handle));
1603 }
1604
1605 /* Mailbox message handler in VF */
1606 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1607         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1608         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1609         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1610         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1611 };
1612
1613 /* Mailbox message handler in PF */
1614 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1615         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1616         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1617         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1618         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1619         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1620         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1621         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1622 };
1623
1624 static int
1625 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1626 {
1627         int err;
1628
1629         /* Initialize mailbox lock */
1630         fm10k_mbx_initlock(hw);
1631
1632         /* Replace default message handler with new ones */
1633         if (hw->mac.type == fm10k_mac_pf)
1634                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1635         else
1636                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1637
1638         if (err) {
1639                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1640                                 err);
1641                 return err;
1642         }
1643         /* Connect to SM for PF device or PF for VF device */
1644         return hw->mbx.ops.connect(hw, &hw->mbx);
1645 }
1646
1647 static void
1648 fm10k_close_mbx_service(struct fm10k_hw *hw)
1649 {
1650         /* Disconnect from SM for PF device or PF for VF device */
1651         hw->mbx.ops.disconnect(hw, &hw->mbx);
1652 }
1653
1654 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1655         .dev_configure          = fm10k_dev_configure,
1656         .dev_start              = fm10k_dev_start,
1657         .dev_stop               = fm10k_dev_stop,
1658         .dev_close              = fm10k_dev_close,
1659         .stats_get              = fm10k_stats_get,
1660         .stats_reset            = fm10k_stats_reset,
1661         .link_update            = fm10k_link_update,
1662         .dev_infos_get          = fm10k_dev_infos_get,
1663         .vlan_filter_set        = fm10k_vlan_filter_set,
1664         .rx_queue_start         = fm10k_dev_rx_queue_start,
1665         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1666         .tx_queue_start         = fm10k_dev_tx_queue_start,
1667         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1668         .rx_queue_setup         = fm10k_rx_queue_setup,
1669         .rx_queue_release       = fm10k_rx_queue_release,
1670         .tx_queue_setup         = fm10k_tx_queue_setup,
1671         .tx_queue_release       = fm10k_tx_queue_release,
1672         .reta_update            = fm10k_reta_update,
1673         .reta_query             = fm10k_reta_query,
1674         .rss_hash_update        = fm10k_rss_hash_update,
1675         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1676 };
1677
1678 static int
1679 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1680 {
1681         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1682         int diag;
1683
1684         PMD_INIT_FUNC_TRACE();
1685
1686         dev->dev_ops = &fm10k_eth_dev_ops;
1687         dev->rx_pkt_burst = &fm10k_recv_pkts;
1688         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1689
1690         if (dev->data->scattered_rx)
1691                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1692
1693         /* only initialize in the primary process */
1694         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1695                 return 0;
1696
1697         /* Vendor and Device ID need to be set before init of shared code */
1698         memset(hw, 0, sizeof(*hw));
1699         hw->device_id = dev->pci_dev->id.device_id;
1700         hw->vendor_id = dev->pci_dev->id.vendor_id;
1701         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1702         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1703         hw->revision_id = 0;
1704         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1705         if (hw->hw_addr == NULL) {
1706                 PMD_INIT_LOG(ERR, "Bad mem resource."
1707                         " Try to blacklist unused devices.");
1708                 return -EIO;
1709         }
1710
1711         /* Store fm10k_adapter pointer */
1712         hw->back = dev->data->dev_private;
1713
1714         /* Initialize the shared code */
1715         diag = fm10k_init_shared_code(hw);
1716         if (diag != FM10K_SUCCESS) {
1717                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1718                 return -EIO;
1719         }
1720
1721         /*
1722          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1723          * there is no way to get link status without reading BAR4.  Until this
1724          * works, assume we have maximum bandwidth.
1725          * @todo - fix bus info
1726          */
1727         hw->bus_caps.speed = fm10k_bus_speed_8000;
1728         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1729         hw->bus_caps.payload = fm10k_bus_payload_512;
1730         hw->bus.speed = fm10k_bus_speed_8000;
1731         hw->bus.width = fm10k_bus_width_pcie_x8;
1732         hw->bus.payload = fm10k_bus_payload_256;
1733
1734         /* Initialize the hw */
1735         diag = fm10k_init_hw(hw);
1736         if (diag != FM10K_SUCCESS) {
1737                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1738                 return -EIO;
1739         }
1740
1741         /* Initialize MAC address(es) */
1742         dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1743         if (dev->data->mac_addrs == NULL) {
1744                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1745                 return -ENOMEM;
1746         }
1747
1748         diag = fm10k_read_mac_addr(hw);
1749         if (diag != FM10K_SUCCESS) {
1750                 /*
1751                  * TODO: remove special handling on VF. Need shared code to
1752                  * fix first.
1753                  */
1754                 if (hw->mac.type == fm10k_mac_pf) {
1755                         PMD_INIT_LOG(ERR, "Read MAC addr failed: %d", diag);
1756                         return -EIO;
1757                 } else {
1758                         /* Generate a random addr */
1759                         eth_random_addr(hw->mac.addr);
1760                         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1761                 }
1762         }
1763
1764         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1765                         &dev->data->mac_addrs[0]);
1766
1767         /* Reset the hw statistics */
1768         fm10k_stats_reset(dev);
1769
1770         /* Reset the hw */
1771         diag = fm10k_reset_hw(hw);
1772         if (diag != FM10K_SUCCESS) {
1773                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1774                 return -EIO;
1775         }
1776
1777         /* Setup mailbox service */
1778         diag = fm10k_setup_mbx_service(hw);
1779         if (diag != FM10K_SUCCESS) {
1780                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1781                 return -EIO;
1782         }
1783
1784         /*PF/VF has different interrupt handling mechanism */
1785         if (hw->mac.type == fm10k_mac_pf) {
1786                 /* register callback func to eal lib */
1787                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1788                         fm10k_dev_interrupt_handler_pf, (void *)dev);
1789
1790                 /* enable MISC interrupt */
1791                 fm10k_dev_enable_intr_pf(dev);
1792         } else { /* VF */
1793                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1794                         fm10k_dev_interrupt_handler_vf, (void *)dev);
1795
1796                 fm10k_dev_enable_intr_vf(dev);
1797         }
1798
1799         /*
1800          * Below function will trigger operations on mailbox, acquire lock to
1801          * avoid race condition from interrupt handler. Operations on mailbox
1802          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
1803          * will handle and generate an interrupt to our side. Then,  FIFO in
1804          * mailbox will be touched.
1805          */
1806         fm10k_mbx_lock(hw);
1807         /* Enable port first */
1808         hw->mac.ops.update_lport_state(hw, 0, 0, 1);
1809
1810         /* Update default vlan */
1811         hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
1812
1813         /*
1814          * Add default mac/vlan filter. glort is assigned by SM for PF, while is
1815          * unused for VF. PF will assign correct glort for VF.
1816          */
1817         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
1818                               hw->mac.default_vid, 1, 0);
1819
1820         /* Set unicast mode by default. App can change to other mode in other
1821          * API func.
1822          */
1823         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1824                                         FM10K_XCAST_MODE_MULTI);
1825
1826         fm10k_mbx_unlock(hw);
1827
1828         /* enable uio intr after callback registered */
1829         rte_intr_enable(&(dev->pci_dev->intr_handle));
1830
1831         return 0;
1832 }
1833
1834 /*
1835  * The set of PCI devices this driver supports. This driver will enable both PF
1836  * and SRIOV-VF devices.
1837  */
1838 static const struct rte_pci_id pci_id_fm10k_map[] = {
1839 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1840 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1841 #include "rte_pci_dev_ids.h"
1842         { .vendor_id = 0, /* sentinel */ },
1843 };
1844
1845 static struct eth_driver rte_pmd_fm10k = {
1846         {
1847                 .name = "rte_pmd_fm10k",
1848                 .id_table = pci_id_fm10k_map,
1849                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1850         },
1851         .eth_dev_init = eth_fm10k_dev_init,
1852         .dev_private_size = sizeof(struct fm10k_adapter),
1853 };
1854
1855 /*
1856  * Driver initialization routine.
1857  * Invoked once at EAL init time.
1858  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
1859  */
1860 static int
1861 rte_pmd_fm10k_init(__rte_unused const char *name,
1862         __rte_unused const char *params)
1863 {
1864         PMD_INIT_FUNC_TRACE();
1865         rte_eth_driver_register(&rte_pmd_fm10k);
1866         return 0;
1867 }
1868
1869 static struct rte_driver rte_fm10k_driver = {
1870         .type = PMD_PDEV,
1871         .init = rte_pmd_fm10k_init,
1872 };
1873
1874 PMD_REGISTER_DRIVER(rte_fm10k_driver);