X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_pmd_fm10k%2Ffm10k_ethdev.c;h=38f6925eef012ccecf634437ac37b089b602ecb6;hb=2d4c1a9ea2ac83aeada1c633f4f205816b904c73;hp=47bfe593214ed6d966508b510ab98a0fdd96a502;hpb=98068e0e044e4a129932ed1c378dd2472bc67f0e;p=dpdk.git diff --git a/lib/librte_pmd_fm10k/fm10k_ethdev.c b/lib/librte_pmd_fm10k/fm10k_ethdev.c index 47bfe59321..38f6925eef 100644 --- a/lib/librte_pmd_fm10k/fm10k_ethdev.c +++ b/lib/librte_pmd_fm10k/fm10k_ethdev.c @@ -44,11 +44,14 @@ #define FM10K_RX_BUFF_ALIGN 512 /* Default delay to acquire mailbox lock */ #define FM10K_MBXLOCK_DELAY_US 20 +#define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL /* Number of chars per uint32 type */ #define CHARS_PER_UINT32 (sizeof(uint32_t)) #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1) +static void fm10k_close_mbx_service(struct fm10k_hw *hw); + static void fm10k_mbx_initlock(struct fm10k_hw *hw) { @@ -68,6 +71,43 @@ fm10k_mbx_unlock(struct fm10k_hw *hw) rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)); } +/* + * reset queue to initial state, allocate software buffers used when starting + * device. + * return 0 on success + * return -ENOMEM if buffers cannot be allocated + * return -EINVAL if buffers do not satisfy alignment condition + */ +static inline int +rx_queue_reset(struct fm10k_rx_queue *q) +{ + uint64_t dma_addr; + int i, diag; + PMD_INIT_FUNC_TRACE(); + + diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc); + if (diag != 0) + return -ENOMEM; + + for (i = 0; i < q->nb_desc; ++i) { + fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id); + if (!fm10k_addr_alignment_valid(q->sw_ring[i])) { + rte_mempool_put_bulk(q->mp, (void **)q->sw_ring, + q->nb_desc); + return -EINVAL; + } + dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]); + q->hw_ring[i].q.pkt_addr = dma_addr; + q->hw_ring[i].q.hdr_addr = dma_addr; + } + + q->next_dd = 0; + q->next_alloc = 0; + q->next_trigger = q->alloc_thresh - 1; + FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1); + return 0; +} + /* * clean queue, descriptor rings, free software buffers used when stopping * device. @@ -108,6 +148,49 @@ rx_queue_free(struct fm10k_rx_queue *q) } } +/* + * disable RX queue, wait unitl HW finished necessary flush operation + */ +static inline int +rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum) +{ + uint32_t reg, i; + + reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum)); + FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum), + reg & ~FM10K_RXQCTL_ENABLE); + + /* Wait 100us at most */ + for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) { + rte_delay_us(1); + reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i)); + if (!(reg & FM10K_RXQCTL_ENABLE)) + break; + } + + if (i == FM10K_QUEUE_DISABLE_TIMEOUT) + return -1; + + return 0; +} + +/* + * reset queue to initial state, allocate software buffers used when starting + * device + */ +static inline void +tx_queue_reset(struct fm10k_tx_queue *q) +{ + PMD_INIT_FUNC_TRACE(); + q->last_free = 0; + q->next_free = 0; + q->nb_used = 0; + q->nb_free = q->nb_desc - 1; + q->free_trigger = q->nb_free - q->free_thresh; + fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh); + FM10K_PCI_REG_WRITE(q->tail_ptr, 0); +} + /* * clean queue, descriptor rings, free software buffers used when stopping * device @@ -150,6 +233,32 @@ tx_queue_free(struct fm10k_tx_queue *q) } } +/* + * disable TX queue, wait unitl HW finished necessary flush operation + */ +static inline int +tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum) +{ + uint32_t reg, i; + + reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum)); + FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum), + reg & ~FM10K_TXDCTL_ENABLE); + + /* Wait 100us at most */ + for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) { + rte_delay_us(1); + reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i)); + if (!(reg & FM10K_TXDCTL_ENABLE)) + break; + } + + if (i == FM10K_QUEUE_DISABLE_TIMEOUT) + return -1; + + return 0; +} + static int fm10k_dev_configure(struct rte_eth_dev *dev) { @@ -161,6 +270,415 @@ fm10k_dev_configure(struct rte_eth_dev *dev) return 0; } +static void +fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_eth_conf *dev_conf = &dev->data->dev_conf; + uint32_t mrqc, *key, i, reta, j; + uint64_t hf; + +#define RSS_KEY_SIZE 40 + static uint8_t rss_intel_key[RSS_KEY_SIZE] = { + 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, + 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, + 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, + 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, + 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, + }; + + if (dev->data->nb_rx_queues == 1 || + dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS || + dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) + return; + + /* random key is rss_intel_key (default) or user provided (rss_key) */ + if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL) + key = (uint32_t *)rss_intel_key; + else + key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key; + + /* Now fill our hash function seeds, 4 bytes at a time */ + for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i) + FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]); + + /* + * Fill in redirection table + * The byte-swap is needed because NIC registers are in + * little-endian order. + */ + reta = 0; + for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) { + if (j == dev->data->nb_rx_queues) + j = 0; + reta = (reta << CHAR_BIT) | j; + if ((i & 3) == 3) + FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), + rte_bswap32(reta)); + } + + /* + * Generate RSS hash based on packet types, TCP/UDP + * port numbers and/or IPv4/v6 src and dst addresses + */ + hf = dev_conf->rx_adv_conf.rss_conf.rss_hf; + mrqc = 0; + mrqc |= (hf & ETH_RSS_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0; + mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0; + mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0; + mrqc |= (hf & ETH_RSS_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0; + + if (mrqc == 0) { + PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not" + "supported", hf); + return; + } + + FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc); +} + +static int +fm10k_dev_tx_init(struct rte_eth_dev *dev) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int i, ret; + struct fm10k_tx_queue *txq; + uint64_t base_addr; + uint32_t size; + + /* Disable TXINT to avoid possible interrupt */ + for (i = 0; i < hw->mac.max_queues; i++) + FM10K_WRITE_REG(hw, FM10K_TXINT(i), + 3 << FM10K_TXINT_TIMER_SHIFT); + + /* Setup TX queue */ + for (i = 0; i < dev->data->nb_tx_queues; ++i) { + txq = dev->data->tx_queues[i]; + base_addr = txq->hw_ring_phys_addr; + size = txq->nb_desc * sizeof(struct fm10k_tx_desc); + + /* disable queue to avoid issues while updating state */ + ret = tx_queue_disable(hw, i); + if (ret) { + PMD_INIT_LOG(ERR, "failed to disable queue %d", i); + return -1; + } + + /* set location and size for descriptor ring */ + FM10K_WRITE_REG(hw, FM10K_TDBAL(i), + base_addr & UINT64_LOWER_32BITS_MASK); + FM10K_WRITE_REG(hw, FM10K_TDBAH(i), + base_addr >> (CHAR_BIT * sizeof(uint32_t))); + FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size); + } + return 0; +} + +static int +fm10k_dev_rx_init(struct rte_eth_dev *dev) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int i, ret; + struct fm10k_rx_queue *rxq; + uint64_t base_addr; + uint32_t size; + uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; + uint16_t buf_size; + struct rte_pktmbuf_pool_private *mbp_priv; + + /* Disable RXINT to avoid possible interrupt */ + for (i = 0; i < hw->mac.max_queues; i++) + FM10K_WRITE_REG(hw, FM10K_RXINT(i), + 3 << FM10K_RXINT_TIMER_SHIFT); + + /* Setup RX queues */ + for (i = 0; i < dev->data->nb_rx_queues; ++i) { + rxq = dev->data->rx_queues[i]; + base_addr = rxq->hw_ring_phys_addr; + size = rxq->nb_desc * sizeof(union fm10k_rx_desc); + + /* disable queue to avoid issues while updating state */ + ret = rx_queue_disable(hw, i); + if (ret) { + PMD_INIT_LOG(ERR, "failed to disable queue %d", i); + return -1; + } + + /* Setup the Base and Length of the Rx Descriptor Ring */ + FM10K_WRITE_REG(hw, FM10K_RDBAL(i), + base_addr & UINT64_LOWER_32BITS_MASK); + FM10K_WRITE_REG(hw, FM10K_RDBAH(i), + base_addr >> (CHAR_BIT * sizeof(uint32_t))); + FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size); + + /* Configure the Rx buffer size for one buff without split */ + mbp_priv = rte_mempool_get_priv(rxq->mp); + buf_size = (uint16_t) (mbp_priv->mbuf_data_room_size - + RTE_PKTMBUF_HEADROOM); + FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), + buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT); + + /* It adds dual VLAN length for supporting dual VLAN */ + if ((dev->data->dev_conf.rxmode.max_rx_pkt_len + + 2 * FM10K_VLAN_TAG_SIZE) > buf_size){ + dev->data->scattered_rx = 1; + dev->rx_pkt_burst = fm10k_recv_scattered_pkts; + } + + /* Enable drop on empty, it's RO for VF */ + if (hw->mac.type == fm10k_mac_pf && rxq->drop_en) + rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; + + FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl); + FM10K_WRITE_FLUSH(hw); + } + + if (dev->data->dev_conf.rxmode.enable_scatter) { + dev->rx_pkt_burst = fm10k_recv_scattered_pkts; + dev->data->scattered_rx = 1; + } + + /* Configure RSS if applicable */ + fm10k_dev_mq_rx_configure(dev); + return 0; +} + +static int +fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int err = -1; + uint32_t reg; + struct fm10k_rx_queue *rxq; + + PMD_INIT_FUNC_TRACE(); + + if (rx_queue_id < dev->data->nb_rx_queues) { + rxq = dev->data->rx_queues[rx_queue_id]; + err = rx_queue_reset(rxq); + if (err == -ENOMEM) { + PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err); + return err; + } else if (err == -EINVAL) { + PMD_INIT_LOG(ERR, "Invalid buffer address alignment :" + " %d", err); + return err; + } + + /* Setup the HW Rx Head and Tail Descriptor Pointers + * Note: this must be done AFTER the queue is enabled on real + * hardware, but BEFORE the queue is enabled when using the + * emulation platform. Do it in both places for now and remove + * this comment and the following two register writes when the + * emulation platform is no longer being used. + */ + FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0); + FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1); + + /* Set PF ownership flag for PF devices */ + reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id)); + if (hw->mac.type == fm10k_mac_pf) + reg |= FM10K_RXQCTL_PF; + reg |= FM10K_RXQCTL_ENABLE; + /* enable RX queue */ + FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg); + FM10K_WRITE_FLUSH(hw); + + /* Setup the HW Rx Head and Tail Descriptor Pointers + * Note: this must be done AFTER the queue is enabled + */ + FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0); + FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1); + } + + return err; +} + +static int +fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + PMD_INIT_FUNC_TRACE(); + + if (rx_queue_id < dev->data->nb_rx_queues) { + /* Disable RX queue */ + rx_queue_disable(hw, rx_queue_id); + + /* Free mbuf and clean HW ring */ + rx_queue_clean(dev->data->rx_queues[rx_queue_id]); + } + + return 0; +} + +static int +fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + /** @todo - this should be defined in the shared code */ +#define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY 0x00010000 + uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY; + int err = 0; + + PMD_INIT_FUNC_TRACE(); + + if (tx_queue_id < dev->data->nb_tx_queues) { + tx_queue_reset(dev->data->tx_queues[tx_queue_id]); + + /* reset head and tail pointers */ + FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0); + FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0); + + /* enable TX queue */ + FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id), + FM10K_TXDCTL_ENABLE | txdctl); + FM10K_WRITE_FLUSH(hw); + } else + err = -1; + + return err; +} + +static int +fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + PMD_INIT_FUNC_TRACE(); + + if (tx_queue_id < dev->data->nb_tx_queues) { + tx_queue_disable(hw, tx_queue_id); + tx_queue_clean(dev->data->tx_queues[tx_queue_id]); + } + + return 0; +} + +/* fls = find last set bit = 32 minus the number of leading zeros */ +#ifndef fls +#define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x)))) +#endif +#define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1) +static int +fm10k_dev_start(struct rte_eth_dev *dev) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int i, diag; + + PMD_INIT_FUNC_TRACE(); + + /* stop, init, then start the hw */ + diag = fm10k_stop_hw(hw); + if (diag != FM10K_SUCCESS) { + PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag); + return -EIO; + } + + diag = fm10k_init_hw(hw); + if (diag != FM10K_SUCCESS) { + PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag); + return -EIO; + } + + diag = fm10k_start_hw(hw); + if (diag != FM10K_SUCCESS) { + PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag); + return -EIO; + } + + diag = fm10k_dev_tx_init(dev); + if (diag) { + PMD_INIT_LOG(ERR, "TX init failed: %d", diag); + return diag; + } + + diag = fm10k_dev_rx_init(dev); + if (diag) { + PMD_INIT_LOG(ERR, "RX init failed: %d", diag); + return diag; + } + + if (hw->mac.type == fm10k_mac_pf) { + /* Establish only VSI 0 as valid */ + FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY); + + /* Configure RSS bits used in RETA table */ + FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), + fls(dev->data->nb_rx_queues - 1) << + FM10K_DGLORTDEC_RSSLENGTH_SHIFT); + + /* Invalidate all other GLORT entries */ + for (i = 1; i < FM10K_DGLORT_COUNT; i++) + FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i), + FM10K_DGLORTMAP_NONE); + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct fm10k_rx_queue *rxq; + rxq = dev->data->rx_queues[i]; + + if (rxq->rx_deferred_start) + continue; + diag = fm10k_dev_rx_queue_start(dev, i); + if (diag != 0) { + int j; + for (j = 0; j < i; ++j) + rx_queue_clean(dev->data->rx_queues[j]); + return diag; + } + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + struct fm10k_tx_queue *txq; + txq = dev->data->tx_queues[i]; + + if (txq->tx_deferred_start) + continue; + diag = fm10k_dev_tx_queue_start(dev, i); + if (diag != 0) { + int j; + for (j = 0; j < dev->data->nb_rx_queues; ++j) + rx_queue_clean(dev->data->rx_queues[j]); + return diag; + } + } + + return 0; +} + +static void +fm10k_dev_stop(struct rte_eth_dev *dev) +{ + int i; + + PMD_INIT_FUNC_TRACE(); + + for (i = 0; i < dev->data->nb_tx_queues; i++) + fm10k_dev_tx_queue_stop(dev, i); + + for (i = 0; i < dev->data->nb_rx_queues; i++) + fm10k_dev_rx_queue_stop(dev, i); +} + +static void +fm10k_dev_close(struct rte_eth_dev *dev) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + PMD_INIT_FUNC_TRACE(); + + /* Stop mailbox service first */ + fm10k_close_mbx_service(hw); + fm10k_dev_stop(dev); + fm10k_stop_hw(hw); +} + static int fm10k_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) @@ -269,6 +787,20 @@ fm10k_dev_infos_get(struct rte_eth_dev *dev, } +static int +fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + PMD_INIT_FUNC_TRACE(); + + /* @todo - add support for the VF */ + if (hw->mac.type != fm10k_mac_pf) + return -ENOTSUP; + + return fm10k_update_vlan(hw, vlan_id, 0, on); +} + static inline int check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request) { @@ -732,6 +1264,336 @@ fm10k_reta_query(struct rte_eth_dev *dev, return 0; } +static int +fm10k_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t *key = (uint32_t *)rss_conf->rss_key; + uint32_t mrqc; + uint64_t hf = rss_conf->rss_hf; + int i; + + PMD_INIT_FUNC_TRACE(); + + if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE * + FM10K_RSSRK_ENTRIES_PER_REG) + return -EINVAL; + + if (hf == 0) + return -EINVAL; + + mrqc = 0; + mrqc |= (hf & ETH_RSS_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0; + mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0; + mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0; + mrqc |= (hf & ETH_RSS_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0; + mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0; + + /* If the mapping doesn't fit any supported, return */ + if (mrqc == 0) + return -EINVAL; + + if (key != NULL) + for (i = 0; i < FM10K_RSSRK_SIZE; ++i) + FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]); + + FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc); + + return 0; +} + +static int +fm10k_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t *key = (uint32_t *)rss_conf->rss_key; + uint32_t mrqc; + uint64_t hf; + int i; + + PMD_INIT_FUNC_TRACE(); + + if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE * + FM10K_RSSRK_ENTRIES_PER_REG) + return -EINVAL; + + if (key != NULL) + for (i = 0; i < FM10K_RSSRK_SIZE; ++i) + key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i)); + + mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0)); + hf = 0; + hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_IPV4_TCP : 0; + hf |= (mrqc & FM10K_MRQC_IPV4) ? ETH_RSS_IPV4 : 0; + hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6 : 0; + hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6_EX : 0; + hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP : 0; + hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX : 0; + hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_IPV4_UDP : 0; + hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP : 0; + hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX : 0; + + rss_conf->rss_hf = hf; + + return 0; +} + +static void +fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t int_map = FM10K_INT_MAP_IMMEDIATE; + + /* Bind all local non-queue interrupt to vector 0 */ + int_map |= 0; + + FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map); + FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map); + FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map); + FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map); + FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map); + FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map); + + /* Enable misc causes */ + FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) | + FM10K_EIMR_ENABLE(THI_FAULT) | + FM10K_EIMR_ENABLE(FUM_FAULT) | + FM10K_EIMR_ENABLE(MAILBOX) | + FM10K_EIMR_ENABLE(SWITCHREADY) | + FM10K_EIMR_ENABLE(SWITCHNOTREADY) | + FM10K_EIMR_ENABLE(SRAMERROR) | + FM10K_EIMR_ENABLE(VFLR)); + + /* Enable ITR 0 */ + FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK | + FM10K_ITR_MASK_CLEAR); + FM10K_WRITE_FLUSH(hw); +} + +static void +fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev) +{ + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t int_map = FM10K_INT_MAP_IMMEDIATE; + + /* Bind all local non-queue interrupt to vector 0 */ + int_map |= 0; + + /* Only INT 0 available, other 15 are reserved. */ + FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map); + + /* Enable ITR 0 */ + FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK | + FM10K_ITR_MASK_CLEAR); + FM10K_WRITE_FLUSH(hw); +} + +static int +fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr) +{ + struct fm10k_fault fault; + int err; + const char *estr = "Unknown error"; + + /* Process PCA fault */ + if (eicr & FM10K_EIMR_PCA_FAULT) { + err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault); + if (err) + goto error; + switch (fault.type) { + case PCA_NO_FAULT: + estr = "PCA_NO_FAULT"; break; + case PCA_UNMAPPED_ADDR: + estr = "PCA_UNMAPPED_ADDR"; break; + case PCA_BAD_QACCESS_PF: + estr = "PCA_BAD_QACCESS_PF"; break; + case PCA_BAD_QACCESS_VF: + estr = "PCA_BAD_QACCESS_VF"; break; + case PCA_MALICIOUS_REQ: + estr = "PCA_MALICIOUS_REQ"; break; + case PCA_POISONED_TLP: + estr = "PCA_POISONED_TLP"; break; + case PCA_TLP_ABORT: + estr = "PCA_TLP_ABORT"; break; + default: + goto error; + } + PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", + estr, fault.func ? "VF" : "PF", fault.func, + fault.address, fault.specinfo); + } + + /* Process THI fault */ + if (eicr & FM10K_EIMR_THI_FAULT) { + err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault); + if (err) + goto error; + switch (fault.type) { + case THI_NO_FAULT: + estr = "THI_NO_FAULT"; break; + case THI_MAL_DIS_Q_FAULT: + estr = "THI_MAL_DIS_Q_FAULT"; break; + default: + goto error; + } + PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", + estr, fault.func ? "VF" : "PF", fault.func, + fault.address, fault.specinfo); + } + + /* Process FUM fault */ + if (eicr & FM10K_EIMR_FUM_FAULT) { + err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault); + if (err) + goto error; + switch (fault.type) { + case FUM_NO_FAULT: + estr = "FUM_NO_FAULT"; break; + case FUM_UNMAPPED_ADDR: + estr = "FUM_UNMAPPED_ADDR"; break; + case FUM_POISONED_TLP: + estr = "FUM_POISONED_TLP"; break; + case FUM_BAD_VF_QACCESS: + estr = "FUM_BAD_VF_QACCESS"; break; + case FUM_ADD_DECODE_ERR: + estr = "FUM_ADD_DECODE_ERR"; break; + case FUM_RO_ERROR: + estr = "FUM_RO_ERROR"; break; + case FUM_QPRC_CRC_ERROR: + estr = "FUM_QPRC_CRC_ERROR"; break; + case FUM_CSR_TIMEOUT: + estr = "FUM_CSR_TIMEOUT"; break; + case FUM_INVALID_TYPE: + estr = "FUM_INVALID_TYPE"; break; + case FUM_INVALID_LENGTH: + estr = "FUM_INVALID_LENGTH"; break; + case FUM_INVALID_BE: + estr = "FUM_INVALID_BE"; break; + case FUM_INVALID_ALIGN: + estr = "FUM_INVALID_ALIGN"; break; + default: + goto error; + } + PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", + estr, fault.func ? "VF" : "PF", fault.func, + fault.address, fault.specinfo); + } + + if (estr) + return 0; + return 0; +error: + PMD_INIT_LOG(ERR, "Failed to handle fault event."); + return err; +} + +/** + * PF interrupt handler triggered by NIC for handling specific interrupt. + * + * @param handle + * Pointer to interrupt handle. + * @param param + * The address of parameter (struct rte_eth_dev *) regsitered before. + * + * @return + * void + */ +static void +fm10k_dev_interrupt_handler_pf( + __rte_unused struct rte_intr_handle *handle, + void *param) +{ + struct rte_eth_dev *dev = (struct rte_eth_dev *)param; + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + uint32_t cause, status; + + if (hw->mac.type != fm10k_mac_pf) + return; + + cause = FM10K_READ_REG(hw, FM10K_EICR); + + /* Handle PCI fault cases */ + if (cause & FM10K_EICR_FAULT_MASK) { + PMD_INIT_LOG(ERR, "INT: find fault!"); + fm10k_dev_handle_fault(hw, cause); + } + + /* Handle switch up/down */ + if (cause & FM10K_EICR_SWITCHNOTREADY) + PMD_INIT_LOG(ERR, "INT: Switch is not ready"); + + if (cause & FM10K_EICR_SWITCHREADY) + PMD_INIT_LOG(INFO, "INT: Switch is ready"); + + /* Handle mailbox message */ + fm10k_mbx_lock(hw); + hw->mbx.ops.process(hw, &hw->mbx); + fm10k_mbx_unlock(hw); + + /* Handle SRAM error */ + if (cause & FM10K_EICR_SRAMERROR) { + PMD_INIT_LOG(ERR, "INT: SRAM error on PEP"); + + status = FM10K_READ_REG(hw, FM10K_SRAM_IP); + /* Write to clear pending bits */ + FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status); + + /* Todo: print out error message after shared code updates */ + } + + /* Clear these 3 events if having any */ + cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX | + FM10K_EICR_SWITCHREADY; + if (cause) + FM10K_WRITE_REG(hw, FM10K_EICR, cause); + + /* Re-enable interrupt from device side */ + FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK | + FM10K_ITR_MASK_CLEAR); + /* Re-enable interrupt from host side */ + rte_intr_enable(&(dev->pci_dev->intr_handle)); +} + +/** + * VF interrupt handler triggered by NIC for handling specific interrupt. + * + * @param handle + * Pointer to interrupt handle. + * @param param + * The address of parameter (struct rte_eth_dev *) regsitered before. + * + * @return + * void + */ +static void +fm10k_dev_interrupt_handler_vf( + __rte_unused struct rte_intr_handle *handle, + void *param) +{ + struct rte_eth_dev *dev = (struct rte_eth_dev *)param; + struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + if (hw->mac.type != fm10k_mac_vf) + return; + + /* Handle mailbox message if lock is acquired */ + fm10k_mbx_lock(hw); + hw->mbx.ops.process(hw, &hw->mbx); + fm10k_mbx_unlock(hw); + + /* Re-enable interrupt from device side */ + FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK | + FM10K_ITR_MASK_CLEAR); + /* Re-enable interrupt from host side */ + rte_intr_enable(&(dev->pci_dev->intr_handle)); +} + /* Mailbox message handler in VF */ static const struct fm10k_msg_data fm10k_msgdata_vf[] = { FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test), @@ -774,18 +1636,35 @@ fm10k_setup_mbx_service(struct fm10k_hw *hw) return hw->mbx.ops.connect(hw, &hw->mbx); } +static void +fm10k_close_mbx_service(struct fm10k_hw *hw) +{ + /* Disconnect from SM for PF device or PF for VF device */ + hw->mbx.ops.disconnect(hw, &hw->mbx); +} + static struct eth_dev_ops fm10k_eth_dev_ops = { .dev_configure = fm10k_dev_configure, + .dev_start = fm10k_dev_start, + .dev_stop = fm10k_dev_stop, + .dev_close = fm10k_dev_close, .stats_get = fm10k_stats_get, .stats_reset = fm10k_stats_reset, .link_update = fm10k_link_update, .dev_infos_get = fm10k_dev_infos_get, + .vlan_filter_set = fm10k_vlan_filter_set, + .rx_queue_start = fm10k_dev_rx_queue_start, + .rx_queue_stop = fm10k_dev_rx_queue_stop, + .tx_queue_start = fm10k_dev_tx_queue_start, + .tx_queue_stop = fm10k_dev_tx_queue_stop, .rx_queue_setup = fm10k_rx_queue_setup, .rx_queue_release = fm10k_rx_queue_release, .tx_queue_setup = fm10k_tx_queue_setup, .tx_queue_release = fm10k_tx_queue_release, .reta_update = fm10k_reta_update, .reta_query = fm10k_reta_query, + .rss_hash_update = fm10k_rss_hash_update, + .rss_hash_conf_get = fm10k_rss_hash_conf_get, }; static int @@ -798,6 +1677,11 @@ eth_fm10k_dev_init(__rte_unused struct eth_driver *eth_drv, PMD_INIT_FUNC_TRACE(); dev->dev_ops = &fm10k_eth_dev_ops; + dev->rx_pkt_burst = &fm10k_recv_pkts; + dev->tx_pkt_burst = &fm10k_xmit_pkts; + + if (dev->data->scattered_rx) + dev->rx_pkt_burst = &fm10k_recv_scattered_pkts; /* only initialize in the primary process */ if (rte_eal_process_type() != RTE_PROC_PRIMARY) @@ -890,6 +1774,21 @@ eth_fm10k_dev_init(__rte_unused struct eth_driver *eth_drv, return -EIO; } + /*PF/VF has different interrupt handling mechanism */ + if (hw->mac.type == fm10k_mac_pf) { + /* register callback func to eal lib */ + rte_intr_callback_register(&(dev->pci_dev->intr_handle), + fm10k_dev_interrupt_handler_pf, (void *)dev); + + /* enable MISC interrupt */ + fm10k_dev_enable_intr_pf(dev); + } else { /* VF */ + rte_intr_callback_register(&(dev->pci_dev->intr_handle), + fm10k_dev_interrupt_handler_vf, (void *)dev); + + fm10k_dev_enable_intr_vf(dev); + } + /* * Below function will trigger operations on mailbox, acquire lock to * avoid race condition from interrupt handler. Operations on mailbox @@ -919,6 +1818,9 @@ eth_fm10k_dev_init(__rte_unused struct eth_driver *eth_drv, fm10k_mbx_unlock(hw); + /* enable uio intr after callback registered */ + rte_intr_enable(&(dev->pci_dev->intr_handle)); + return 0; } @@ -928,6 +1830,7 @@ eth_fm10k_dev_init(__rte_unused struct eth_driver *eth_drv, */ static struct rte_pci_id pci_id_fm10k_map[] = { #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) }, +#define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) }, #include "rte_pci_dev_ids.h" { .vendor_id = 0, /* sentinel */ }, };