lib/librte_pmd_fm10k/fm10k_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <rte_ethdev.h>
  35 #include <rte_malloc.h>
  36 #include <rte_memzone.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_dev.h>
  39 #include <rte_spinlock.h>
  40
  41 #include "fm10k.h"
  42 #include "base/fm10k_api.h"
  43
  44 #define FM10K_RX_BUFF_ALIGN 512
  45 /* Default delay to acquire mailbox lock */
  46 #define FM10K_MBXLOCK_DELAY_US 20
  47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
  48
  49 /* Number of chars per uint32 type */
  50 #define CHARS_PER_UINT32 (sizeof(uint32_t))
  51 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
  52
  53 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
  54
  55 static void
  56 fm10k_mbx_initlock(struct fm10k_hw *hw)
  57 {
  58         rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  59 }
  60
  61 static void
  62 fm10k_mbx_lock(struct fm10k_hw *hw)
  63 {
  64         while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
  65                 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
  66 }
  67
  68 static void
  69 fm10k_mbx_unlock(struct fm10k_hw *hw)
  70 {
  71         rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
  72 }
  73
  74 /*
  75  * reset queue to initial state, allocate software buffers used when starting
  76  * device.
  77  * return 0 on success
  78  * return -ENOMEM if buffers cannot be allocated
  79  * return -EINVAL if buffers do not satisfy alignment condition
  80  */
  81 static inline int
  82 rx_queue_reset(struct fm10k_rx_queue *q)
  83 {
  84         uint64_t dma_addr;
  85         int i, diag;
  86         PMD_INIT_FUNC_TRACE();
  87
  88         diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
  89         if (diag != 0)
  90                 return -ENOMEM;
  91
  92         for (i = 0; i < q->nb_desc; ++i) {
  93                 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
  94                 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
  95                         rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
  96                                                 q->nb_desc);
  97                         return -EINVAL;
  98                 }
  99                 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
 100                 q->hw_ring[i].q.pkt_addr = dma_addr;
 101                 q->hw_ring[i].q.hdr_addr = dma_addr;
 102         }
 103
 104         q->next_dd = 0;
 105         q->next_alloc = 0;
 106         q->next_trigger = q->alloc_thresh - 1;
 107         FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
 108         return 0;
 109 }
 110
 111 /*
 112  * clean queue, descriptor rings, free software buffers used when stopping
 113  * device.
 114  */
 115 static inline void
 116 rx_queue_clean(struct fm10k_rx_queue *q)
 117 {
 118         union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
 119         uint32_t i;
 120         PMD_INIT_FUNC_TRACE();
 121
 122         /* zero descriptor rings */
 123         for (i = 0; i < q->nb_desc; ++i)
 124                 q->hw_ring[i] = zero;
 125
 126         /* free software buffers */
 127         for (i = 0; i < q->nb_desc; ++i) {
 128                 if (q->sw_ring[i]) {
 129                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 130                         q->sw_ring[i] = NULL;
 131                 }
 132         }
 133 }
 134
 135 /*
 136  * free all queue memory used when releasing the queue (i.e. configure)
 137  */
 138 static inline void
 139 rx_queue_free(struct fm10k_rx_queue *q)
 140 {
 141         PMD_INIT_FUNC_TRACE();
 142         if (q) {
 143                 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
 144                 rx_queue_clean(q);
 145                 if (q->sw_ring) {
 146                         rte_free(q->sw_ring);
 147                         q->sw_ring = NULL;
 148                 }
 149                 rte_free(q);
 150                 q = NULL;
 151         }
 152 }
 153
 154 /*
 155  * disable RX queue, wait unitl HW finished necessary flush operation
 156  */
 157 static inline int
 158 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 159 {
 160         uint32_t reg, i;
 161
 162         reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
 163         FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
 164                         reg & ~FM10K_RXQCTL_ENABLE);
 165
 166         /* Wait 100us at most */
 167         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 168                 rte_delay_us(1);
 169                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
 170                 if (!(reg & FM10K_RXQCTL_ENABLE))
 171                         break;
 172         }
 173
 174         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 175                 return -1;
 176
 177         return 0;
 178 }
 179
 180 /*
 181  * reset queue to initial state, allocate software buffers used when starting
 182  * device
 183  */
 184 static inline void
 185 tx_queue_reset(struct fm10k_tx_queue *q)
 186 {
 187         PMD_INIT_FUNC_TRACE();
 188         q->last_free = 0;
 189         q->next_free = 0;
 190         q->nb_used = 0;
 191         q->nb_free = q->nb_desc - 1;
 192         q->free_trigger = q->nb_free - q->free_thresh;
 193         fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
 194         FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
 195 }
 196
 197 /*
 198  * clean queue, descriptor rings, free software buffers used when stopping
 199  * device
 200  */
 201 static inline void
 202 tx_queue_clean(struct fm10k_tx_queue *q)
 203 {
 204         struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
 205         uint32_t i;
 206         PMD_INIT_FUNC_TRACE();
 207
 208         /* zero descriptor rings */
 209         for (i = 0; i < q->nb_desc; ++i)
 210                 q->hw_ring[i] = zero;
 211
 212         /* free software buffers */
 213         for (i = 0; i < q->nb_desc; ++i) {
 214                 if (q->sw_ring[i]) {
 215                         rte_pktmbuf_free_seg(q->sw_ring[i]);
 216                         q->sw_ring[i] = NULL;
 217                 }
 218         }
 219 }
 220
 221 /*
 222  * free all queue memory used when releasing the queue (i.e. configure)
 223  */
 224 static inline void
 225 tx_queue_free(struct fm10k_tx_queue *q)
 226 {
 227         PMD_INIT_FUNC_TRACE();
 228         if (q) {
 229                 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
 230                 tx_queue_clean(q);
 231                 if (q->rs_tracker.list) {
 232                         rte_free(q->rs_tracker.list);
 233                         q->rs_tracker.list = NULL;
 234                 }
 235                 if (q->sw_ring) {
 236                         rte_free(q->sw_ring);
 237                         q->sw_ring = NULL;
 238                 }
 239                 rte_free(q);
 240                 q = NULL;
 241         }
 242 }
 243
 244 /*
 245  * disable TX queue, wait unitl HW finished necessary flush operation
 246  */
 247 static inline int
 248 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
 249 {
 250         uint32_t reg, i;
 251
 252         reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
 253         FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
 254                         reg & ~FM10K_TXDCTL_ENABLE);
 255
 256         /* Wait 100us at most */
 257         for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
 258                 rte_delay_us(1);
 259                 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
 260                 if (!(reg & FM10K_TXDCTL_ENABLE))
 261                         break;
 262         }
 263
 264         if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
 265                 return -1;
 266
 267         return 0;
 268 }
 269
 270 static int
 271 fm10k_dev_configure(struct rte_eth_dev *dev)
 272 {
 273         PMD_INIT_FUNC_TRACE();
 274
 275         if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
 276                 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
 277
 278         return 0;
 279 }
 280
 281 static void
 282 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
 283 {
 284         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 285         struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 286         uint32_t mrqc, *key, i, reta, j;
 287         uint64_t hf;
 288
 289 #define RSS_KEY_SIZE 40
 290         static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
 291                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 292                 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 293                 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 294                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 295                 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 296         };
 297
 298         if (dev->data->nb_rx_queues == 1 ||
 299             dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
 300             dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
 301                 return;
 302
 303         /* random key is rss_intel_key (default) or user provided (rss_key) */
 304         if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
 305                 key = (uint32_t *)rss_intel_key;
 306         else
 307                 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
 308
 309         /* Now fill our hash function seeds, 4 bytes at a time */
 310         for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
 311                 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
 312
 313         /*
 314          * Fill in redirection table
 315          * The byte-swap is needed because NIC registers are in
 316          * little-endian order.
 317          */
 318         reta = 0;
 319         for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
 320                 if (j == dev->data->nb_rx_queues)
 321                         j = 0;
 322                 reta = (reta << CHAR_BIT) | j;
 323                 if ((i & 3) == 3)
 324                         FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
 325                                         rte_bswap32(reta));
 326         }
 327
 328         /*
 329          * Generate RSS hash based on packet types, TCP/UDP
 330          * port numbers and/or IPv4/v6 src and dst addresses
 331          */
 332         hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
 333         mrqc = 0;
 334         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
 335         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
 336         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
 337         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
 338         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
 339         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
 340         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
 341         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
 342         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
 343
 344         if (mrqc == 0) {
 345                 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
 346                         "supported", hf);
 347                 return;
 348         }
 349
 350         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
 351 }
 352
 353 static int
 354 fm10k_dev_tx_init(struct rte_eth_dev *dev)
 355 {
 356         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 357         int i, ret;
 358         struct fm10k_tx_queue *txq;
 359         uint64_t base_addr;
 360         uint32_t size;
 361
 362         /* Disable TXINT to avoid possible interrupt */
 363         for (i = 0; i < hw->mac.max_queues; i++)
 364                 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
 365                                 3 << FM10K_TXINT_TIMER_SHIFT);
 366
 367         /* Setup TX queue */
 368         for (i = 0; i < dev->data->nb_tx_queues; ++i) {
 369                 txq = dev->data->tx_queues[i];
 370                 base_addr = txq->hw_ring_phys_addr;
 371                 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
 372
 373                 /* disable queue to avoid issues while updating state */
 374                 ret = tx_queue_disable(hw, i);
 375                 if (ret) {
 376                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 377                         return -1;
 378                 }
 379
 380                 /* set location and size for descriptor ring */
 381                 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
 382                                 base_addr & UINT64_LOWER_32BITS_MASK);
 383                 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
 384                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 385                 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
 386         }
 387         return 0;
 388 }
 389
 390 static int
 391 fm10k_dev_rx_init(struct rte_eth_dev *dev)
 392 {
 393         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 394         int i, ret;
 395         struct fm10k_rx_queue *rxq;
 396         uint64_t base_addr;
 397         uint32_t size;
 398         uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 399         uint16_t buf_size;
 400
 401         /* Disable RXINT to avoid possible interrupt */
 402         for (i = 0; i < hw->mac.max_queues; i++)
 403                 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
 404                                 3 << FM10K_RXINT_TIMER_SHIFT);
 405
 406         /* Setup RX queues */
 407         for (i = 0; i < dev->data->nb_rx_queues; ++i) {
 408                 rxq = dev->data->rx_queues[i];
 409                 base_addr = rxq->hw_ring_phys_addr;
 410                 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
 411
 412                 /* disable queue to avoid issues while updating state */
 413                 ret = rx_queue_disable(hw, i);
 414                 if (ret) {
 415                         PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
 416                         return -1;
 417                 }
 418
 419                 /* Setup the Base and Length of the Rx Descriptor Ring */
 420                 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
 421                                 base_addr & UINT64_LOWER_32BITS_MASK);
 422                 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
 423                                 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
 424                 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
 425
 426                 /* Configure the Rx buffer size for one buff without split */
 427                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
 428                         RTE_PKTMBUF_HEADROOM);
 429                 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
 430                                 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
 431
 432                 /* It adds dual VLAN length for supporting dual VLAN */
 433                 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
 434                                 2 * FM10K_VLAN_TAG_SIZE) > buf_size){
 435                         dev->data->scattered_rx = 1;
 436                         dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 437                 }
 438
 439                 /* Enable drop on empty, it's RO for VF */
 440                 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
 441                         rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
 442
 443                 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
 444                 FM10K_WRITE_FLUSH(hw);
 445         }
 446
 447         if (dev->data->dev_conf.rxmode.enable_scatter) {
 448                 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
 449                 dev->data->scattered_rx = 1;
 450         }
 451
 452         /* Configure RSS if applicable */
 453         fm10k_dev_mq_rx_configure(dev);
 454         return 0;
 455 }
 456
 457 static int
 458 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 459 {
 460         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 461         int err = -1;
 462         uint32_t reg;
 463         struct fm10k_rx_queue *rxq;
 464
 465         PMD_INIT_FUNC_TRACE();
 466
 467         if (rx_queue_id < dev->data->nb_rx_queues) {
 468                 rxq = dev->data->rx_queues[rx_queue_id];
 469                 err = rx_queue_reset(rxq);
 470                 if (err == -ENOMEM) {
 471                         PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
 472                         return err;
 473                 } else if (err == -EINVAL) {
 474                         PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
 475                                 " %d", err);
 476                         return err;
 477                 }
 478
 479                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 480                  * Note: this must be done AFTER the queue is enabled on real
 481                  * hardware, but BEFORE the queue is enabled when using the
 482                  * emulation platform. Do it in both places for now and remove
 483                  * this comment and the following two register writes when the
 484                  * emulation platform is no longer being used.
 485                  */
 486                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 487                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 488
 489                 /* Set PF ownership flag for PF devices */
 490                 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
 491                 if (hw->mac.type == fm10k_mac_pf)
 492                         reg |= FM10K_RXQCTL_PF;
 493                 reg |= FM10K_RXQCTL_ENABLE;
 494                 /* enable RX queue */
 495                 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
 496                 FM10K_WRITE_FLUSH(hw);
 497
 498                 /* Setup the HW Rx Head and Tail Descriptor Pointers
 499                  * Note: this must be done AFTER the queue is enabled
 500                  */
 501                 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
 502                 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
 503         }
 504
 505         return err;
 506 }
 507
 508 static int
 509 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 510 {
 511         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 512
 513         PMD_INIT_FUNC_TRACE();
 514
 515         if (rx_queue_id < dev->data->nb_rx_queues) {
 516                 /* Disable RX queue */
 517                 rx_queue_disable(hw, rx_queue_id);
 518
 519                 /* Free mbuf and clean HW ring */
 520                 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
 521         }
 522
 523         return 0;
 524 }
 525
 526 static int
 527 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 528 {
 529         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 530         /** @todo - this should be defined in the shared code */
 531 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY       0x00010000
 532         uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
 533         int err = 0;
 534
 535         PMD_INIT_FUNC_TRACE();
 536
 537         if (tx_queue_id < dev->data->nb_tx_queues) {
 538                 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
 539
 540                 /* reset head and tail pointers */
 541                 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
 542                 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
 543
 544                 /* enable TX queue */
 545                 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
 546                                         FM10K_TXDCTL_ENABLE | txdctl);
 547                 FM10K_WRITE_FLUSH(hw);
 548         } else
 549                 err = -1;
 550
 551         return err;
 552 }
 553
 554 static int
 555 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 556 {
 557         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 558
 559         PMD_INIT_FUNC_TRACE();
 560
 561         if (tx_queue_id < dev->data->nb_tx_queues) {
 562                 tx_queue_disable(hw, tx_queue_id);
 563                 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
 564         }
 565
 566         return 0;
 567 }
 568
 569 /* fls = find last set bit = 32 minus the number of leading zeros */
 570 #ifndef fls
 571 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
 572 #endif
 573 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
 574 static int
 575 fm10k_dev_start(struct rte_eth_dev *dev)
 576 {
 577         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 578         int i, diag;
 579
 580         PMD_INIT_FUNC_TRACE();
 581
 582         /* stop, init, then start the hw */
 583         diag = fm10k_stop_hw(hw);
 584         if (diag != FM10K_SUCCESS) {
 585                 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
 586                 return -EIO;
 587         }
 588
 589         diag = fm10k_init_hw(hw);
 590         if (diag != FM10K_SUCCESS) {
 591                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
 592                 return -EIO;
 593         }
 594
 595         diag = fm10k_start_hw(hw);
 596         if (diag != FM10K_SUCCESS) {
 597                 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
 598                 return -EIO;
 599         }
 600
 601         diag = fm10k_dev_tx_init(dev);
 602         if (diag) {
 603                 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
 604                 return diag;
 605         }
 606
 607         diag = fm10k_dev_rx_init(dev);
 608         if (diag) {
 609                 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
 610                 return diag;
 611         }
 612
 613         if (hw->mac.type == fm10k_mac_pf) {
 614                 /* Establish only VSI 0 as valid */
 615                 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
 616
 617                 /* Configure RSS bits used in RETA table */
 618                 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
 619                                 fls(dev->data->nb_rx_queues - 1) <<
 620                                 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
 621
 622                 /* Invalidate all other GLORT entries */
 623                 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
 624                         FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
 625                                         FM10K_DGLORTMAP_NONE);
 626         }
 627
 628         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 629                 struct fm10k_rx_queue *rxq;
 630                 rxq = dev->data->rx_queues[i];
 631
 632                 if (rxq->rx_deferred_start)
 633                         continue;
 634                 diag = fm10k_dev_rx_queue_start(dev, i);
 635                 if (diag != 0) {
 636                         int j;
 637                         for (j = 0; j < i; ++j)
 638                                 rx_queue_clean(dev->data->rx_queues[j]);
 639                         return diag;
 640                 }
 641         }
 642
 643         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 644                 struct fm10k_tx_queue *txq;
 645                 txq = dev->data->tx_queues[i];
 646
 647                 if (txq->tx_deferred_start)
 648                         continue;
 649                 diag = fm10k_dev_tx_queue_start(dev, i);
 650                 if (diag != 0) {
 651                         int j;
 652                         for (j = 0; j < dev->data->nb_rx_queues; ++j)
 653                                 rx_queue_clean(dev->data->rx_queues[j]);
 654                         return diag;
 655                 }
 656         }
 657
 658         return 0;
 659 }
 660
 661 static void
 662 fm10k_dev_stop(struct rte_eth_dev *dev)
 663 {
 664         int i;
 665
 666         PMD_INIT_FUNC_TRACE();
 667
 668         for (i = 0; i < dev->data->nb_tx_queues; i++)
 669                 fm10k_dev_tx_queue_stop(dev, i);
 670
 671         for (i = 0; i < dev->data->nb_rx_queues; i++)
 672                 fm10k_dev_rx_queue_stop(dev, i);
 673 }
 674
 675 static void
 676 fm10k_dev_close(struct rte_eth_dev *dev)
 677 {
 678         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 679
 680         PMD_INIT_FUNC_TRACE();
 681
 682         /* Stop mailbox service first */
 683         fm10k_close_mbx_service(hw);
 684         fm10k_dev_stop(dev);
 685         fm10k_stop_hw(hw);
 686 }
 687
 688 static int
 689 fm10k_link_update(struct rte_eth_dev *dev,
 690         __rte_unused int wait_to_complete)
 691 {
 692         PMD_INIT_FUNC_TRACE();
 693
 694         /* The host-interface link is always up.  The speed is ~50Gbps per Gen3
 695          * x8 PCIe interface. For now, we leave the speed undefined since there
 696          * is no 50Gbps Ethernet. */
 697         dev->data->dev_link.link_speed  = 0;
 698         dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 699         dev->data->dev_link.link_status = 1;
 700
 701         return 0;
 702 }
 703
 704 static void
 705 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 706 {
 707         uint64_t ipackets, opackets, ibytes, obytes;
 708         struct fm10k_hw *hw =
 709                 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 710         struct fm10k_hw_stats *hw_stats =
 711                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 712         int i;
 713
 714         PMD_INIT_FUNC_TRACE();
 715
 716         fm10k_update_hw_stats(hw, hw_stats);
 717
 718         ipackets = opackets = ibytes = obytes = 0;
 719         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
 720                 (i < FM10K_MAX_QUEUES_PF); ++i) {
 721                 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
 722                 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
 723                 stats->q_ibytes[i]   = hw_stats->q[i].rx_bytes.count;
 724                 stats->q_obytes[i]   = hw_stats->q[i].tx_bytes.count;
 725                 ipackets += stats->q_ipackets[i];
 726                 opackets += stats->q_opackets[i];
 727                 ibytes   += stats->q_ibytes[i];
 728                 obytes   += stats->q_obytes[i];
 729         }
 730         stats->ipackets = ipackets;
 731         stats->opackets = opackets;
 732         stats->ibytes = ibytes;
 733         stats->obytes = obytes;
 734 }
 735
 736 static void
 737 fm10k_stats_reset(struct rte_eth_dev *dev)
 738 {
 739         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 740         struct fm10k_hw_stats *hw_stats =
 741                 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 742
 743         PMD_INIT_FUNC_TRACE();
 744
 745         memset(hw_stats, 0, sizeof(*hw_stats));
 746         fm10k_rebind_hw_stats(hw, hw_stats);
 747 }
 748
 749 static void
 750 fm10k_dev_infos_get(struct rte_eth_dev *dev,
 751         struct rte_eth_dev_info *dev_info)
 752 {
 753         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 754
 755         PMD_INIT_FUNC_TRACE();
 756
 757         dev_info->min_rx_bufsize     = FM10K_MIN_RX_BUF_SIZE;
 758         dev_info->max_rx_pktlen      = FM10K_MAX_PKT_SIZE;
 759         dev_info->max_rx_queues      = hw->mac.max_queues;
 760         dev_info->max_tx_queues      = hw->mac.max_queues;
 761         dev_info->max_mac_addrs      = 1;
 762         dev_info->max_hash_mac_addrs = 0;
 763         dev_info->max_vfs            = FM10K_MAX_VF_NUM;
 764         dev_info->max_vmdq_pools     = ETH_64_POOLS;
 765         dev_info->rx_offload_capa =
 766                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 767                 DEV_RX_OFFLOAD_UDP_CKSUM  |
 768                 DEV_RX_OFFLOAD_TCP_CKSUM;
 769         dev_info->tx_offload_capa    = 0;
 770         dev_info->reta_size = FM10K_MAX_RSS_INDICES;
 771
 772         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 773                 .rx_thresh = {
 774                         .pthresh = FM10K_DEFAULT_RX_PTHRESH,
 775                         .hthresh = FM10K_DEFAULT_RX_HTHRESH,
 776                         .wthresh = FM10K_DEFAULT_RX_WTHRESH,
 777                 },
 778                 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
 779                 .rx_drop_en = 0,
 780         };
 781
 782         dev_info->default_txconf = (struct rte_eth_txconf) {
 783                 .tx_thresh = {
 784                         .pthresh = FM10K_DEFAULT_TX_PTHRESH,
 785                         .hthresh = FM10K_DEFAULT_TX_HTHRESH,
 786                         .wthresh = FM10K_DEFAULT_TX_WTHRESH,
 787                 },
 788                 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
 789                 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
 790                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 791                                 ETH_TXQ_FLAGS_NOOFFLOADS,
 792         };
 793
 794 }
 795
 796 static int
 797 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 798 {
 799         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 800
 801         PMD_INIT_FUNC_TRACE();
 802
 803         /* @todo - add support for the VF */
 804         if (hw->mac.type != fm10k_mac_pf)
 805                 return -ENOTSUP;
 806
 807         return fm10k_update_vlan(hw, vlan_id, 0, on);
 808 }
 809
 810 static inline int
 811 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
 812 {
 813         if ((request < min) || (request > max) || ((request % mult) != 0))
 814                 return -1;
 815         else
 816                 return 0;
 817 }
 818
 819 /*
 820  * Create a memzone for hardware descriptor rings. Malloc cannot be used since
 821  * the physical address is required. If the memzone is already created, then
 822  * this function returns a pointer to the existing memzone.
 823  */
 824 static inline const struct rte_memzone *
 825 allocate_hw_ring(const char *driver_name, const char *ring_name,
 826         uint8_t port_id, uint16_t queue_id, int socket_id,
 827         uint32_t size, uint32_t align)
 828 {
 829         char name[RTE_MEMZONE_NAMESIZE];
 830         const struct rte_memzone *mz;
 831
 832         snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
 833                  driver_name, ring_name, port_id, queue_id, socket_id);
 834
 835         /* return the memzone if it already exists */
 836         mz = rte_memzone_lookup(name);
 837         if (mz)
 838                 return mz;
 839
 840 #ifdef RTE_LIBRTE_XEN_DOM0
 841         return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
 842                                            RTE_PGSIZE_2M);
 843 #else
 844         return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
 845 #endif
 846 }
 847
 848 static inline int
 849 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
 850 {
 851         if ((request < min) || (request > max) || ((div % request) != 0))
 852                 return -1;
 853         else
 854                 return 0;
 855 }
 856
 857 static inline int
 858 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
 859 {
 860         uint16_t rx_free_thresh;
 861
 862         if (conf->rx_free_thresh == 0)
 863                 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
 864         else
 865                 rx_free_thresh = conf->rx_free_thresh;
 866
 867         /* make sure the requested threshold satisfies the constraints */
 868         if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
 869                         FM10K_RX_FREE_THRESH_MAX(q),
 870                         FM10K_RX_FREE_THRESH_DIV(q),
 871                         rx_free_thresh)) {
 872                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
 873                         "less than or equal to %u, "
 874                         "greater than or equal to %u, "
 875                         "and a divisor of %u",
 876                         rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
 877                         FM10K_RX_FREE_THRESH_MIN(q),
 878                         FM10K_RX_FREE_THRESH_DIV(q));
 879                 return (-EINVAL);
 880         }
 881
 882         q->alloc_thresh = rx_free_thresh;
 883         q->drop_en = conf->rx_drop_en;
 884         q->rx_deferred_start = conf->rx_deferred_start;
 885
 886         return 0;
 887 }
 888
 889 /*
 890  * Hardware requires specific alignment for Rx packet buffers. At
 891  * least one of the following two conditions must be satisfied.
 892  *  1. Address is 512B aligned
 893  *  2. Address is 8B aligned and buffer does not cross 4K boundary.
 894  *
 895  * As such, the driver may need to adjust the DMA address within the
 896  * buffer by up to 512B. The mempool element size is checked here
 897  * to make sure a maximally sized Ethernet frame can still be wholly
 898  * contained within the buffer after 512B alignment.
 899  *
 900  * return 1 if the element size is valid, otherwise return 0.
 901  */
 902 static int
 903 mempool_element_size_valid(struct rte_mempool *mp)
 904 {
 905         uint32_t min_size;
 906
 907         /* elt_size includes mbuf header and headroom */
 908         min_size = mp->elt_size - sizeof(struct rte_mbuf) -
 909                         RTE_PKTMBUF_HEADROOM;
 910
 911         /* account for up to 512B of alignment */
 912         min_size -= FM10K_RX_BUFF_ALIGN;
 913
 914         /* sanity check for overflow */
 915         if (min_size > mp->elt_size)
 916                 return 0;
 917
 918         if (min_size < ETHER_MAX_VLAN_FRAME_LEN)
 919                 return 0;
 920
 921         /* size is valid */
 922         return 1;
 923 }
 924
 925 static int
 926 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
 927         uint16_t nb_desc, unsigned int socket_id,
 928         const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
 929 {
 930         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 931         struct fm10k_rx_queue *q;
 932         const struct rte_memzone *mz;
 933
 934         PMD_INIT_FUNC_TRACE();
 935
 936         /* make sure the mempool element size can account for alignment. */
 937         if (!mempool_element_size_valid(mp)) {
 938                 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
 939                 return (-EINVAL);
 940         }
 941
 942         /* make sure a valid number of descriptors have been requested */
 943         if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
 944                                 FM10K_MULT_RX_DESC, nb_desc)) {
 945                 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
 946                         "less than or equal to %"PRIu32", "
 947                         "greater than or equal to %u, "
 948                         "and a multiple of %u",
 949                         nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
 950                         FM10K_MULT_RX_DESC);
 951                 return (-EINVAL);
 952         }
 953
 954         /*
 955          * if this queue existed already, free the associated memory. The
 956          * queue cannot be reused in case we need to allocate memory on
 957          * different socket than was previously used.
 958          */
 959         if (dev->data->rx_queues[queue_id] != NULL) {
 960                 rx_queue_free(dev->data->rx_queues[queue_id]);
 961                 dev->data->rx_queues[queue_id] = NULL;
 962         }
 963
 964         /* allocate memory for the queue structure */
 965         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
 966                                 socket_id);
 967         if (q == NULL) {
 968                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
 969                 return (-ENOMEM);
 970         }
 971
 972         /* setup queue */
 973         q->mp = mp;
 974         q->nb_desc = nb_desc;
 975         q->port_id = dev->data->port_id;
 976         q->queue_id = queue_id;
 977         q->tail_ptr = (volatile uint32_t *)
 978                 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
 979         if (handle_rxconf(q, conf))
 980                 return (-EINVAL);
 981
 982         /* allocate memory for the software ring */
 983         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
 984                                         nb_desc * sizeof(struct rte_mbuf *),
 985                                         RTE_CACHE_LINE_SIZE, socket_id);
 986         if (q->sw_ring == NULL) {
 987                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
 988                 rte_free(q);
 989                 return (-ENOMEM);
 990         }
 991
 992         /*
 993          * allocate memory for the hardware descriptor ring. A memzone large
 994          * enough to hold the maximum ring size is requested to allow for
 995          * resizing in later calls to the queue setup function.
 996          */
 997         mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
 998                                 dev->data->port_id, queue_id, socket_id,
 999                                 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1000         if (mz == NULL) {
1001                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1002                 rte_free(q->sw_ring);
1003                 rte_free(q);
1004                 return (-ENOMEM);
1005         }
1006         q->hw_ring = mz->addr;
1007         q->hw_ring_phys_addr = mz->phys_addr;
1008
1009         dev->data->rx_queues[queue_id] = q;
1010         return 0;
1011 }
1012
1013 static void
1014 fm10k_rx_queue_release(void *queue)
1015 {
1016         PMD_INIT_FUNC_TRACE();
1017
1018         rx_queue_free(queue);
1019 }
1020
1021 static inline int
1022 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1023 {
1024         uint16_t tx_free_thresh;
1025         uint16_t tx_rs_thresh;
1026
1027         /* constraint MACROs require that tx_free_thresh is configured
1028          * before tx_rs_thresh */
1029         if (conf->tx_free_thresh == 0)
1030                 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1031         else
1032                 tx_free_thresh = conf->tx_free_thresh;
1033
1034         /* make sure the requested threshold satisfies the constraints */
1035         if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1036                         FM10K_TX_FREE_THRESH_MAX(q),
1037                         FM10K_TX_FREE_THRESH_DIV(q),
1038                         tx_free_thresh)) {
1039                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1040                         "less than or equal to %u, "
1041                         "greater than or equal to %u, "
1042                         "and a divisor of %u",
1043                         tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1044                         FM10K_TX_FREE_THRESH_MIN(q),
1045                         FM10K_TX_FREE_THRESH_DIV(q));
1046                 return (-EINVAL);
1047         }
1048
1049         q->free_thresh = tx_free_thresh;
1050
1051         if (conf->tx_rs_thresh == 0)
1052                 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1053         else
1054                 tx_rs_thresh = conf->tx_rs_thresh;
1055
1056         q->tx_deferred_start = conf->tx_deferred_start;
1057
1058         /* make sure the requested threshold satisfies the constraints */
1059         if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1060                         FM10K_TX_RS_THRESH_MAX(q),
1061                         FM10K_TX_RS_THRESH_DIV(q),
1062                         tx_rs_thresh)) {
1063                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1064                         "less than or equal to %u, "
1065                         "greater than or equal to %u, "
1066                         "and a divisor of %u",
1067                         tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1068                         FM10K_TX_RS_THRESH_MIN(q),
1069                         FM10K_TX_RS_THRESH_DIV(q));
1070                 return (-EINVAL);
1071         }
1072
1073         q->rs_thresh = tx_rs_thresh;
1074
1075         return 0;
1076 }
1077
1078 static int
1079 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1080         uint16_t nb_desc, unsigned int socket_id,
1081         const struct rte_eth_txconf *conf)
1082 {
1083         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1084         struct fm10k_tx_queue *q;
1085         const struct rte_memzone *mz;
1086
1087         PMD_INIT_FUNC_TRACE();
1088
1089         /* make sure a valid number of descriptors have been requested */
1090         if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1091                                 FM10K_MULT_TX_DESC, nb_desc)) {
1092                 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1093                         "less than or equal to %"PRIu32", "
1094                         "greater than or equal to %u, "
1095                         "and a multiple of %u",
1096                         nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1097                         FM10K_MULT_TX_DESC);
1098                 return (-EINVAL);
1099         }
1100
1101         /*
1102          * if this queue existed already, free the associated memory. The
1103          * queue cannot be reused in case we need to allocate memory on
1104          * different socket than was previously used.
1105          */
1106         if (dev->data->tx_queues[queue_id] != NULL) {
1107                 tx_queue_free(dev->data->tx_queues[queue_id]);
1108                 dev->data->tx_queues[queue_id] = NULL;
1109         }
1110
1111         /* allocate memory for the queue structure */
1112         q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1113                                 socket_id);
1114         if (q == NULL) {
1115                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1116                 return (-ENOMEM);
1117         }
1118
1119         /* setup queue */
1120         q->nb_desc = nb_desc;
1121         q->port_id = dev->data->port_id;
1122         q->queue_id = queue_id;
1123         q->tail_ptr = (volatile uint32_t *)
1124                 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1125         if (handle_txconf(q, conf))
1126                 return (-EINVAL);
1127
1128         /* allocate memory for the software ring */
1129         q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1130                                         nb_desc * sizeof(struct rte_mbuf *),
1131                                         RTE_CACHE_LINE_SIZE, socket_id);
1132         if (q->sw_ring == NULL) {
1133                 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1134                 rte_free(q);
1135                 return (-ENOMEM);
1136         }
1137
1138         /*
1139          * allocate memory for the hardware descriptor ring. A memzone large
1140          * enough to hold the maximum ring size is requested to allow for
1141          * resizing in later calls to the queue setup function.
1142          */
1143         mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1144                                 dev->data->port_id, queue_id, socket_id,
1145                                 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1146         if (mz == NULL) {
1147                 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1148                 rte_free(q->sw_ring);
1149                 rte_free(q);
1150                 return (-ENOMEM);
1151         }
1152         q->hw_ring = mz->addr;
1153         q->hw_ring_phys_addr = mz->phys_addr;
1154
1155         /*
1156          * allocate memory for the RS bit tracker. Enough slots to hold the
1157          * descriptor index for each RS bit needing to be set are required.
1158          */
1159         q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1160                                 ((nb_desc + 1) / q->rs_thresh) *
1161                                 sizeof(uint16_t),
1162                                 RTE_CACHE_LINE_SIZE, socket_id);
1163         if (q->rs_tracker.list == NULL) {
1164                 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1165                 rte_free(q->sw_ring);
1166                 rte_free(q);
1167                 return (-ENOMEM);
1168         }
1169
1170         dev->data->tx_queues[queue_id] = q;
1171         return 0;
1172 }
1173
1174 static void
1175 fm10k_tx_queue_release(void *queue)
1176 {
1177         PMD_INIT_FUNC_TRACE();
1178
1179         tx_queue_free(queue);
1180 }
1181
1182 static int
1183 fm10k_reta_update(struct rte_eth_dev *dev,
1184                         struct rte_eth_rss_reta_entry64 *reta_conf,
1185                         uint16_t reta_size)
1186 {
1187         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1188         uint16_t i, j, idx, shift;
1189         uint8_t mask;
1190         uint32_t reta;
1191
1192         PMD_INIT_FUNC_TRACE();
1193
1194         if (reta_size > FM10K_MAX_RSS_INDICES) {
1195                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1196                         "(%d) doesn't match the number hardware can supported "
1197                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1198                 return -EINVAL;
1199         }
1200
1201         /*
1202          * Update Redirection Table RETA[n], n=0..31. The redirection table has
1203          * 128-entries in 32 registers
1204          */
1205         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1206                 idx = i / RTE_RETA_GROUP_SIZE;
1207                 shift = i % RTE_RETA_GROUP_SIZE;
1208                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1209                                 BIT_MASK_PER_UINT32);
1210                 if (mask == 0)
1211                         continue;
1212
1213                 reta = 0;
1214                 if (mask != BIT_MASK_PER_UINT32)
1215                         reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1216
1217                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1218                         if (mask & (0x1 << j)) {
1219                                 if (mask != 0xF)
1220                                         reta &= ~(UINT8_MAX << CHAR_BIT * j);
1221                                 reta |= reta_conf[idx].reta[shift + j] <<
1222                                                 (CHAR_BIT * j);
1223                         }
1224                 }
1225                 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1226         }
1227
1228         return 0;
1229 }
1230
1231 static int
1232 fm10k_reta_query(struct rte_eth_dev *dev,
1233                         struct rte_eth_rss_reta_entry64 *reta_conf,
1234                         uint16_t reta_size)
1235 {
1236         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1237         uint16_t i, j, idx, shift;
1238         uint8_t mask;
1239         uint32_t reta;
1240
1241         PMD_INIT_FUNC_TRACE();
1242
1243         if (reta_size < FM10K_MAX_RSS_INDICES) {
1244                 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1245                         "(%d) doesn't match the number hardware can supported "
1246                         "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1247                 return -EINVAL;
1248         }
1249
1250         /*
1251          * Read Redirection Table RETA[n], n=0..31. The redirection table has
1252          * 128-entries in 32 registers
1253          */
1254         for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1255                 idx = i / RTE_RETA_GROUP_SIZE;
1256                 shift = i % RTE_RETA_GROUP_SIZE;
1257                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1258                                 BIT_MASK_PER_UINT32);
1259                 if (mask == 0)
1260                         continue;
1261
1262                 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1263                 for (j = 0; j < CHARS_PER_UINT32; j++) {
1264                         if (mask & (0x1 << j))
1265                                 reta_conf[idx].reta[shift + j] = ((reta >>
1266                                         CHAR_BIT * j) & UINT8_MAX);
1267                 }
1268         }
1269
1270         return 0;
1271 }
1272
1273 static int
1274 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1275         struct rte_eth_rss_conf *rss_conf)
1276 {
1277         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1278         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1279         uint32_t mrqc;
1280         uint64_t hf = rss_conf->rss_hf;
1281         int i;
1282
1283         PMD_INIT_FUNC_TRACE();
1284
1285         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1286                 FM10K_RSSRK_ENTRIES_PER_REG)
1287                 return -EINVAL;
1288
1289         if (hf == 0)
1290                 return -EINVAL;
1291
1292         mrqc = 0;
1293         mrqc |= (hf & ETH_RSS_IPV4)              ? FM10K_MRQC_IPV4     : 0;
1294         mrqc |= (hf & ETH_RSS_IPV6)              ? FM10K_MRQC_IPV6     : 0;
1295         mrqc |= (hf & ETH_RSS_IPV6_EX)           ? FM10K_MRQC_IPV6     : 0;
1296         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? FM10K_MRQC_TCP_IPV4 : 0;
1297         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? FM10K_MRQC_TCP_IPV6 : 0;
1298         mrqc |= (hf & ETH_RSS_IPV6_TCP_EX)       ? FM10K_MRQC_TCP_IPV6 : 0;
1299         mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP)  ? FM10K_MRQC_UDP_IPV4 : 0;
1300         mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP)  ? FM10K_MRQC_UDP_IPV6 : 0;
1301         mrqc |= (hf & ETH_RSS_IPV6_UDP_EX)       ? FM10K_MRQC_UDP_IPV6 : 0;
1302
1303         /* If the mapping doesn't fit any supported, return */
1304         if (mrqc == 0)
1305                 return -EINVAL;
1306
1307         if (key != NULL)
1308                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1309                         FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1310
1311         FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1312
1313         return 0;
1314 }
1315
1316 static int
1317 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1318         struct rte_eth_rss_conf *rss_conf)
1319 {
1320         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1321         uint32_t *key = (uint32_t *)rss_conf->rss_key;
1322         uint32_t mrqc;
1323         uint64_t hf;
1324         int i;
1325
1326         PMD_INIT_FUNC_TRACE();
1327
1328         if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1329                                 FM10K_RSSRK_ENTRIES_PER_REG)
1330                 return -EINVAL;
1331
1332         if (key != NULL)
1333                 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1334                         key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1335
1336         mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1337         hf = 0;
1338         hf |= (mrqc & FM10K_MRQC_IPV4)     ? ETH_RSS_IPV4              : 0;
1339         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6              : 0;
1340         hf |= (mrqc & FM10K_MRQC_IPV6)     ? ETH_RSS_IPV6_EX           : 0;
1341         hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP  : 0;
1342         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP  : 0;
1343         hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX       : 0;
1344         hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP  : 0;
1345         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP  : 0;
1346         hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX       : 0;
1347
1348         rss_conf->rss_hf = hf;
1349
1350         return 0;
1351 }
1352
1353 static void
1354 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1355 {
1356         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1357         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1358
1359         /* Bind all local non-queue interrupt to vector 0 */
1360         int_map |= 0;
1361
1362         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1363         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1364         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1365         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1366         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1367         FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1368
1369         /* Enable misc causes */
1370         FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1371                                 FM10K_EIMR_ENABLE(THI_FAULT) |
1372                                 FM10K_EIMR_ENABLE(FUM_FAULT) |
1373                                 FM10K_EIMR_ENABLE(MAILBOX) |
1374                                 FM10K_EIMR_ENABLE(SWITCHREADY) |
1375                                 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1376                                 FM10K_EIMR_ENABLE(SRAMERROR) |
1377                                 FM10K_EIMR_ENABLE(VFLR));
1378
1379         /* Enable ITR 0 */
1380         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1381                                         FM10K_ITR_MASK_CLEAR);
1382         FM10K_WRITE_FLUSH(hw);
1383 }
1384
1385 static void
1386 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1387 {
1388         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1389         uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1390
1391         /* Bind all local non-queue interrupt to vector 0 */
1392         int_map |= 0;
1393
1394         /* Only INT 0 available, other 15 are reserved. */
1395         FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1396
1397         /* Enable ITR 0 */
1398         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1399                                         FM10K_ITR_MASK_CLEAR);
1400         FM10K_WRITE_FLUSH(hw);
1401 }
1402
1403 static int
1404 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1405 {
1406         struct fm10k_fault fault;
1407         int err;
1408         const char *estr = "Unknown error";
1409
1410         /* Process PCA fault */
1411         if (eicr & FM10K_EIMR_PCA_FAULT) {
1412                 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1413                 if (err)
1414                         goto error;
1415                 switch (fault.type) {
1416                 case PCA_NO_FAULT:
1417                         estr = "PCA_NO_FAULT"; break;
1418                 case PCA_UNMAPPED_ADDR:
1419                         estr = "PCA_UNMAPPED_ADDR"; break;
1420                 case PCA_BAD_QACCESS_PF:
1421                         estr = "PCA_BAD_QACCESS_PF"; break;
1422                 case PCA_BAD_QACCESS_VF:
1423                         estr = "PCA_BAD_QACCESS_VF"; break;
1424                 case PCA_MALICIOUS_REQ:
1425                         estr = "PCA_MALICIOUS_REQ"; break;
1426                 case PCA_POISONED_TLP:
1427                         estr = "PCA_POISONED_TLP"; break;
1428                 case PCA_TLP_ABORT:
1429                         estr = "PCA_TLP_ABORT"; break;
1430                 default:
1431                         goto error;
1432                 }
1433                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1434                         estr, fault.func ? "VF" : "PF", fault.func,
1435                         fault.address, fault.specinfo);
1436         }
1437
1438         /* Process THI fault */
1439         if (eicr & FM10K_EIMR_THI_FAULT) {
1440                 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1441                 if (err)
1442                         goto error;
1443                 switch (fault.type) {
1444                 case THI_NO_FAULT:
1445                         estr = "THI_NO_FAULT"; break;
1446                 case THI_MAL_DIS_Q_FAULT:
1447                         estr = "THI_MAL_DIS_Q_FAULT"; break;
1448                 default:
1449                         goto error;
1450                 }
1451                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1452                         estr, fault.func ? "VF" : "PF", fault.func,
1453                         fault.address, fault.specinfo);
1454         }
1455
1456         /* Process FUM fault */
1457         if (eicr & FM10K_EIMR_FUM_FAULT) {
1458                 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1459                 if (err)
1460                         goto error;
1461                 switch (fault.type) {
1462                 case FUM_NO_FAULT:
1463                         estr = "FUM_NO_FAULT"; break;
1464                 case FUM_UNMAPPED_ADDR:
1465                         estr = "FUM_UNMAPPED_ADDR"; break;
1466                 case FUM_POISONED_TLP:
1467                         estr = "FUM_POISONED_TLP"; break;
1468                 case FUM_BAD_VF_QACCESS:
1469                         estr = "FUM_BAD_VF_QACCESS"; break;
1470                 case FUM_ADD_DECODE_ERR:
1471                         estr = "FUM_ADD_DECODE_ERR"; break;
1472                 case FUM_RO_ERROR:
1473                         estr = "FUM_RO_ERROR"; break;
1474                 case FUM_QPRC_CRC_ERROR:
1475                         estr = "FUM_QPRC_CRC_ERROR"; break;
1476                 case FUM_CSR_TIMEOUT:
1477                         estr = "FUM_CSR_TIMEOUT"; break;
1478                 case FUM_INVALID_TYPE:
1479                         estr = "FUM_INVALID_TYPE"; break;
1480                 case FUM_INVALID_LENGTH:
1481                         estr = "FUM_INVALID_LENGTH"; break;
1482                 case FUM_INVALID_BE:
1483                         estr = "FUM_INVALID_BE"; break;
1484                 case FUM_INVALID_ALIGN:
1485                         estr = "FUM_INVALID_ALIGN"; break;
1486                 default:
1487                         goto error;
1488                 }
1489                 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1490                         estr, fault.func ? "VF" : "PF", fault.func,
1491                         fault.address, fault.specinfo);
1492         }
1493
1494         if (estr)
1495                 return 0;
1496         return 0;
1497 error:
1498         PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1499         return err;
1500 }
1501
1502 /**
1503  * PF interrupt handler triggered by NIC for handling specific interrupt.
1504  *
1505  * @param handle
1506  *  Pointer to interrupt handle.
1507  * @param param
1508  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1509  *
1510  * @return
1511  *  void
1512  */
1513 static void
1514 fm10k_dev_interrupt_handler_pf(
1515                         __rte_unused struct rte_intr_handle *handle,
1516                         void *param)
1517 {
1518         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1519         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1520         uint32_t cause, status;
1521
1522         if (hw->mac.type != fm10k_mac_pf)
1523                 return;
1524
1525         cause = FM10K_READ_REG(hw, FM10K_EICR);
1526
1527         /* Handle PCI fault cases */
1528         if (cause & FM10K_EICR_FAULT_MASK) {
1529                 PMD_INIT_LOG(ERR, "INT: find fault!");
1530                 fm10k_dev_handle_fault(hw, cause);
1531         }
1532
1533         /* Handle switch up/down */
1534         if (cause & FM10K_EICR_SWITCHNOTREADY)
1535                 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1536
1537         if (cause & FM10K_EICR_SWITCHREADY)
1538                 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1539
1540         /* Handle mailbox message */
1541         fm10k_mbx_lock(hw);
1542         hw->mbx.ops.process(hw, &hw->mbx);
1543         fm10k_mbx_unlock(hw);
1544
1545         /* Handle SRAM error */
1546         if (cause & FM10K_EICR_SRAMERROR) {
1547                 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1548
1549                 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1550                 /* Write to clear pending bits */
1551                 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1552
1553                 /* Todo: print out error message after shared code  updates */
1554         }
1555
1556         /* Clear these 3 events if having any */
1557         cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1558                  FM10K_EICR_SWITCHREADY;
1559         if (cause)
1560                 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1561
1562         /* Re-enable interrupt from device side */
1563         FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1564                                         FM10K_ITR_MASK_CLEAR);
1565         /* Re-enable interrupt from host side */
1566         rte_intr_enable(&(dev->pci_dev->intr_handle));
1567 }
1568
1569 /**
1570  * VF interrupt handler triggered by NIC for handling specific interrupt.
1571  *
1572  * @param handle
1573  *  Pointer to interrupt handle.
1574  * @param param
1575  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1576  *
1577  * @return
1578  *  void
1579  */
1580 static void
1581 fm10k_dev_interrupt_handler_vf(
1582                         __rte_unused struct rte_intr_handle *handle,
1583                         void *param)
1584 {
1585         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1586         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1587
1588         if (hw->mac.type != fm10k_mac_vf)
1589                 return;
1590
1591         /* Handle mailbox message if lock is acquired */
1592         fm10k_mbx_lock(hw);
1593         hw->mbx.ops.process(hw, &hw->mbx);
1594         fm10k_mbx_unlock(hw);
1595
1596         /* Re-enable interrupt from device side */
1597         FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1598                                         FM10K_ITR_MASK_CLEAR);
1599         /* Re-enable interrupt from host side */
1600         rte_intr_enable(&(dev->pci_dev->intr_handle));
1601 }
1602
1603 /* Mailbox message handler in VF */
1604 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1605         FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1606         FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1607         FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1608         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1609 };
1610
1611 /* Mailbox message handler in PF */
1612 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1613         FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1614         FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1615         FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1616         FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1617         FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1618         FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1619         FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1620 };
1621
1622 static int
1623 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1624 {
1625         int err;
1626
1627         /* Initialize mailbox lock */
1628         fm10k_mbx_initlock(hw);
1629
1630         /* Replace default message handler with new ones */
1631         if (hw->mac.type == fm10k_mac_pf)
1632                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1633         else
1634                 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1635
1636         if (err) {
1637                 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1638                                 err);
1639                 return err;
1640         }
1641         /* Connect to SM for PF device or PF for VF device */
1642         return hw->mbx.ops.connect(hw, &hw->mbx);
1643 }
1644
1645 static void
1646 fm10k_close_mbx_service(struct fm10k_hw *hw)
1647 {
1648         /* Disconnect from SM for PF device or PF for VF device */
1649         hw->mbx.ops.disconnect(hw, &hw->mbx);
1650 }
1651
1652 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1653         .dev_configure          = fm10k_dev_configure,
1654         .dev_start              = fm10k_dev_start,
1655         .dev_stop               = fm10k_dev_stop,
1656         .dev_close              = fm10k_dev_close,
1657         .stats_get              = fm10k_stats_get,
1658         .stats_reset            = fm10k_stats_reset,
1659         .link_update            = fm10k_link_update,
1660         .dev_infos_get          = fm10k_dev_infos_get,
1661         .vlan_filter_set        = fm10k_vlan_filter_set,
1662         .rx_queue_start         = fm10k_dev_rx_queue_start,
1663         .rx_queue_stop          = fm10k_dev_rx_queue_stop,
1664         .tx_queue_start         = fm10k_dev_tx_queue_start,
1665         .tx_queue_stop          = fm10k_dev_tx_queue_stop,
1666         .rx_queue_setup         = fm10k_rx_queue_setup,
1667         .rx_queue_release       = fm10k_rx_queue_release,
1668         .tx_queue_setup         = fm10k_tx_queue_setup,
1669         .tx_queue_release       = fm10k_tx_queue_release,
1670         .reta_update            = fm10k_reta_update,
1671         .reta_query             = fm10k_reta_query,
1672         .rss_hash_update        = fm10k_rss_hash_update,
1673         .rss_hash_conf_get      = fm10k_rss_hash_conf_get,
1674 };
1675
1676 static int
1677 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1678 {
1679         struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1680         int diag;
1681
1682         PMD_INIT_FUNC_TRACE();
1683
1684         dev->dev_ops = &fm10k_eth_dev_ops;
1685         dev->rx_pkt_burst = &fm10k_recv_pkts;
1686         dev->tx_pkt_burst = &fm10k_xmit_pkts;
1687
1688         if (dev->data->scattered_rx)
1689                 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1690
1691         /* only initialize in the primary process */
1692         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1693                 return 0;
1694
1695         /* Vendor and Device ID need to be set before init of shared code */
1696         memset(hw, 0, sizeof(*hw));
1697         hw->device_id = dev->pci_dev->id.device_id;
1698         hw->vendor_id = dev->pci_dev->id.vendor_id;
1699         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1700         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1701         hw->revision_id = 0;
1702         hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1703         if (hw->hw_addr == NULL) {
1704                 PMD_INIT_LOG(ERR, "Bad mem resource."
1705                         " Try to blacklist unused devices.");
1706                 return -EIO;
1707         }
1708
1709         /* Store fm10k_adapter pointer */
1710         hw->back = dev->data->dev_private;
1711
1712         /* Initialize the shared code */
1713         diag = fm10k_init_shared_code(hw);
1714         if (diag != FM10K_SUCCESS) {
1715                 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1716                 return -EIO;
1717         }
1718
1719         /*
1720          * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1721          * there is no way to get link status without reading BAR4.  Until this
1722          * works, assume we have maximum bandwidth.
1723          * @todo - fix bus info
1724          */
1725         hw->bus_caps.speed = fm10k_bus_speed_8000;
1726         hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1727         hw->bus_caps.payload = fm10k_bus_payload_512;
1728         hw->bus.speed = fm10k_bus_speed_8000;
1729         hw->bus.width = fm10k_bus_width_pcie_x8;
1730         hw->bus.payload = fm10k_bus_payload_256;
1731
1732         /* Initialize the hw */
1733         diag = fm10k_init_hw(hw);
1734         if (diag != FM10K_SUCCESS) {
1735                 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1736                 return -EIO;
1737         }
1738
1739         /* Initialize MAC address(es) */
1740         dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1741         if (dev->data->mac_addrs == NULL) {
1742                 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1743                 return -ENOMEM;
1744         }
1745
1746         diag = fm10k_read_mac_addr(hw);
1747         if (diag != FM10K_SUCCESS) {
1748                 /*
1749                  * TODO: remove special handling on VF. Need shared code to
1750                  * fix first.
1751                  */
1752                 if (hw->mac.type == fm10k_mac_pf) {
1753                         PMD_INIT_LOG(ERR, "Read MAC addr failed: %d", diag);
1754                         return -EIO;
1755                 } else {
1756                         /* Generate a random addr */
1757                         eth_random_addr(hw->mac.addr);
1758                         memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1759                 }
1760         }
1761
1762         ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1763                         &dev->data->mac_addrs[0]);
1764
1765         /* Reset the hw statistics */
1766         fm10k_stats_reset(dev);
1767
1768         /* Reset the hw */
1769         diag = fm10k_reset_hw(hw);
1770         if (diag != FM10K_SUCCESS) {
1771                 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1772                 return -EIO;
1773         }
1774
1775         /* Setup mailbox service */
1776         diag = fm10k_setup_mbx_service(hw);
1777         if (diag != FM10K_SUCCESS) {
1778                 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1779                 return -EIO;
1780         }
1781
1782         /*PF/VF has different interrupt handling mechanism */
1783         if (hw->mac.type == fm10k_mac_pf) {
1784                 /* register callback func to eal lib */
1785                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1786                         fm10k_dev_interrupt_handler_pf, (void *)dev);
1787
1788                 /* enable MISC interrupt */
1789                 fm10k_dev_enable_intr_pf(dev);
1790         } else { /* VF */
1791                 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1792                         fm10k_dev_interrupt_handler_vf, (void *)dev);
1793
1794                 fm10k_dev_enable_intr_vf(dev);
1795         }
1796
1797         /*
1798          * Below function will trigger operations on mailbox, acquire lock to
1799          * avoid race condition from interrupt handler. Operations on mailbox
1800          * FIFO will trigger interrupt to PF/SM, in which interrupt handler
1801          * will handle and generate an interrupt to our side. Then,  FIFO in
1802          * mailbox will be touched.
1803          */
1804         fm10k_mbx_lock(hw);
1805         /* Enable port first */
1806         hw->mac.ops.update_lport_state(hw, 0, 0, 1);
1807
1808         /* Update default vlan */
1809         hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
1810
1811         /*
1812          * Add default mac/vlan filter. glort is assigned by SM for PF, while is
1813          * unused for VF. PF will assign correct glort for VF.
1814          */
1815         hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
1816                               hw->mac.default_vid, 1, 0);
1817
1818         /* Set unicast mode by default. App can change to other mode in other
1819          * API func.
1820          */
1821         hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1822                                         FM10K_XCAST_MODE_MULTI);
1823
1824         fm10k_mbx_unlock(hw);
1825
1826         /* enable uio intr after callback registered */
1827         rte_intr_enable(&(dev->pci_dev->intr_handle));
1828
1829         return 0;
1830 }
1831
1832 /*
1833  * The set of PCI devices this driver supports. This driver will enable both PF
1834  * and SRIOV-VF devices.
1835  */
1836 static const struct rte_pci_id pci_id_fm10k_map[] = {
1837 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1838 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1839 #include "rte_pci_dev_ids.h"
1840         { .vendor_id = 0, /* sentinel */ },
1841 };
1842
1843 static struct eth_driver rte_pmd_fm10k = {
1844         {
1845                 .name = "rte_pmd_fm10k",
1846                 .id_table = pci_id_fm10k_map,
1847                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1848         },
1849         .eth_dev_init = eth_fm10k_dev_init,
1850         .dev_private_size = sizeof(struct fm10k_adapter),
1851 };
1852
1853 /*
1854  * Driver initialization routine.
1855  * Invoked once at EAL init time.
1856  * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
1857  */
1858 static int
1859 rte_pmd_fm10k_init(__rte_unused const char *name,
1860         __rte_unused const char *params)
1861 {
1862         PMD_INIT_FUNC_TRACE();
1863         rte_eth_driver_register(&rte_pmd_fm10k);
1864         return 0;
1865 }
1866
1867 static struct rte_driver rte_fm10k_driver = {
1868         .type = PMD_PDEV,
1869         .init = rte_pmd_fm10k_init,
1870 };
1871
1872 PMD_REGISTER_DRIVER(rte_fm10k_driver);