X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Ftxgbe%2Ftxgbe_rxtx.c;h=a70bf0b9c61ecb59da56823ea7053855284993df;hb=e413a125512c213089e283a2702f155fe75ff7b2;hp=dc6af74fab6fbec30ed5a9e77af757e06166fe07;hpb=db9767a58380fbe43b8457e8dd59f49e12e56055;p=dpdk.git diff --git a/drivers/net/txgbe/txgbe_rxtx.c b/drivers/net/txgbe/txgbe_rxtx.c index dc6af74fab..a70bf0b9c6 100644 --- a/drivers/net/txgbe/txgbe_rxtx.c +++ b/drivers/net/txgbe/txgbe_rxtx.c @@ -19,7 +19,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -40,6 +41,12 @@ #include "txgbe_ethdev.h" #include "txgbe_rxtx.h" +#ifdef RTE_LIBRTE_IEEE1588 +#define TXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST +#else +#define TXGBE_TX_IEEE1588_TMST 0 +#endif + /* Bit Mask to indicate what bits required for building TX context */ static const u64 TXGBE_TX_OFFLOAD_MASK = (PKT_TX_IP_CKSUM | PKT_TX_OUTER_IPV6 | @@ -50,7 +57,11 @@ static const u64 TXGBE_TX_OFFLOAD_MASK = (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK | PKT_TX_TCP_SEG | PKT_TX_TUNNEL_MASK | - PKT_TX_OUTER_IP_CKSUM); + PKT_TX_OUTER_IP_CKSUM | +#ifdef RTE_LIB_SECURITY + PKT_TX_SEC_OFFLOAD | +#endif + TXGBE_TX_IEEE1588_TMST); #define TXGBE_TX_OFFLOAD_NOTSUP_MASK \ (PKT_TX_OFFLOAD_MASK ^ TXGBE_TX_OFFLOAD_MASK) @@ -304,7 +315,8 @@ txgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, static inline void txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq, volatile struct txgbe_tx_ctx_desc *ctx_txd, - uint64_t ol_flags, union txgbe_tx_offload tx_offload) + uint64_t ol_flags, union txgbe_tx_offload tx_offload, + __rte_unused uint64_t *mdata) { union txgbe_tx_offload tx_offload_mask; uint32_t type_tucmd_mlhl; @@ -398,6 +410,19 @@ txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq, vlan_macip_lens |= TXGBE_TXD_VLAN(tx_offload.vlan_tci); } +#ifdef RTE_LIB_SECURITY + if (ol_flags & PKT_TX_SEC_OFFLOAD) { + union txgbe_crypto_tx_desc_md *md = + (union txgbe_crypto_tx_desc_md *)mdata; + tunnel_seed |= TXGBE_TXD_IPSEC_SAIDX(md->sa_idx); + type_tucmd_mlhl |= md->enc ? + (TXGBE_TXD_IPSEC_ESP | TXGBE_TXD_IPSEC_ESPENC) : 0; + type_tucmd_mlhl |= TXGBE_TXD_IPSEC_ESPLEN(md->pad_len); + tx_offload_mask.sa_idx |= ~0; + tx_offload_mask.sec_pad_len |= ~0; + } +#endif + txq->ctx_cache[ctx_idx].flags = ol_flags; txq->ctx_cache[ctx_idx].tx_offload.data[0] = tx_offload_mask.data[0] & tx_offload.data[0]; @@ -694,6 +719,9 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint32_t ctx = 0; uint32_t new_ctx; union txgbe_tx_offload tx_offload; +#ifdef RTE_LIB_SECURITY + uint8_t use_ipsec; +#endif tx_offload.data[0] = 0; tx_offload.data[1] = 0; @@ -720,6 +748,9 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * are needed for offload functionality. */ ol_flags = tx_pkt->ol_flags; +#ifdef RTE_LIB_SECURITY + use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD); +#endif /* If hardware offload required */ tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK; @@ -735,6 +766,16 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, tx_offload.outer_l3_len = tx_pkt->outer_l3_len; tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt); +#ifdef RTE_LIB_SECURITY + if (use_ipsec) { + union txgbe_crypto_tx_desc_md *ipsec_mdata = + (union txgbe_crypto_tx_desc_md *) + rte_security_dynfield(tx_pkt); + tx_offload.sa_idx = ipsec_mdata->sa_idx; + tx_offload.sec_pad_len = ipsec_mdata->pad_len; + } +#endif + /* If new context need be built or reuse the exist ctx*/ ctx = what_ctx_update(txq, tx_ol_req, tx_offload); /* Only allocate context descriptor if required */ @@ -851,6 +892,11 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, */ cmd_type_len = TXGBE_TXD_FCS; +#ifdef RTE_LIBRTE_IEEE1588 + if (ol_flags & PKT_TX_IEEE1588_TMST) + cmd_type_len |= TXGBE_TXD_1588; +#endif + olinfo_status = 0; if (tx_ol_req) { if (ol_flags & PKT_TX_TCP_SEG) { @@ -883,7 +929,8 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } txgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, - tx_offload); + tx_offload, + rte_security_dynfield(tx_pkt)); txe->last_id = tx_last; tx_id = txe->next_id; @@ -902,6 +949,10 @@ txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } olinfo_status |= TXGBE_TXD_PAYLEN(pkt_len); +#ifdef RTE_LIB_SECURITY + if (use_ipsec) + olinfo_status |= TXGBE_TXD_IPSEC; +#endif m_seg = tx_pkt; do { @@ -1028,8 +1079,20 @@ txgbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) PKT_RX_RSS_HASH, 0, 0, 0, 0, 0, 0, PKT_RX_FDIR, }; - +#ifdef RTE_LIBRTE_IEEE1588 + static uint64_t ip_pkt_etqf_map[8] = { + 0, 0, 0, PKT_RX_IEEE1588_PTP, + 0, 0, 0, 0, + }; + int etfid = txgbe_etflt_id(TXGBE_RXD_PTID(pkt_info)); + if (likely(-1 != etfid)) + return ip_pkt_etqf_map[etfid] | + ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; + else + return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; +#else return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; +#endif } static inline uint64_t @@ -1046,6 +1109,10 @@ rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) vlan_flags & PKT_RX_VLAN_STRIPPED) ? vlan_flags : 0; +#ifdef RTE_LIBRTE_IEEE1588 + if (rx_status & TXGBE_RXD_STAT_1588) + pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST; +#endif return pkt_flags; } @@ -1067,8 +1134,16 @@ rx_desc_error_to_pkt_flags(uint32_t rx_status) if (rx_status & TXGBE_RXD_STAT_EIPCS && rx_status & TXGBE_RXD_ERR_EIPCS) { - pkt_flags |= PKT_RX_EIP_CKSUM_BAD; + pkt_flags |= PKT_RX_OUTER_IP_CKSUM_BAD; + } + +#ifdef RTE_LIB_SECURITY + if (rx_status & TXGBE_RXD_STAT_SECP) { + pkt_flags |= PKT_RX_SEC_OFFLOAD; + if (rx_status & TXGBE_RXD_ERR_SECERR) + pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED; } +#endif return pkt_flags; } @@ -1117,7 +1192,7 @@ txgbe_rx_scan_hw_ring(struct txgbe_rx_queue *rxq) for (j = 0; j < LOOK_AHEAD; j++) s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status); - rte_smp_rmb(); + rte_atomic_thread_fence(__ATOMIC_ACQUIRE); /* Compute how many status bits were set */ for (nb_dd = 0; nb_dd < LOOK_AHEAD && @@ -1898,6 +1973,11 @@ txgbe_get_rx_port_offloads(struct rte_eth_dev *dev) offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; +#ifdef RTE_LIB_SECURITY + if (dev->security_ctx) + offloads |= DEV_RX_OFFLOAD_SECURITY; +#endif + return offloads; } @@ -1916,6 +1996,98 @@ txgbe_tx_queue_release_mbufs(struct txgbe_tx_queue *txq) } } +static int +txgbe_tx_done_cleanup_full(struct txgbe_tx_queue *txq, uint32_t free_cnt) +{ + struct txgbe_tx_entry *swr_ring = txq->sw_ring; + uint16_t i, tx_last, tx_id; + uint16_t nb_tx_free_last; + uint16_t nb_tx_to_clean; + uint32_t pkt_cnt; + + /* Start free mbuf from the next of tx_tail */ + tx_last = txq->tx_tail; + tx_id = swr_ring[tx_last].next_id; + + if (txq->nb_tx_free == 0 && txgbe_xmit_cleanup(txq)) + return 0; + + nb_tx_to_clean = txq->nb_tx_free; + nb_tx_free_last = txq->nb_tx_free; + if (!free_cnt) + free_cnt = txq->nb_tx_desc; + + /* Loop through swr_ring to count the amount of + * freeable mubfs and packets. + */ + for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { + for (i = 0; i < nb_tx_to_clean && + pkt_cnt < free_cnt && + tx_id != tx_last; i++) { + if (swr_ring[tx_id].mbuf != NULL) { + rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); + swr_ring[tx_id].mbuf = NULL; + + /* + * last segment in the packet, + * increment packet count + */ + pkt_cnt += (swr_ring[tx_id].last_id == tx_id); + } + + tx_id = swr_ring[tx_id].next_id; + } + + if (pkt_cnt < free_cnt) { + if (txgbe_xmit_cleanup(txq)) + break; + + nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; + nb_tx_free_last = txq->nb_tx_free; + } + } + + return (int)pkt_cnt; +} + +static int +txgbe_tx_done_cleanup_simple(struct txgbe_tx_queue *txq, + uint32_t free_cnt) +{ + int i, n, cnt; + + if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) + free_cnt = txq->nb_tx_desc; + + cnt = free_cnt - free_cnt % txq->tx_free_thresh; + + for (i = 0; i < cnt; i += n) { + if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh) + break; + + n = txgbe_tx_free_bufs(txq); + + if (n == 0) + break; + } + + return i; +} + +int +txgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) +{ + struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue; + if (txq->offloads == 0 && +#ifdef RTE_LIB_SECURITY + !(txq->using_ipsec) && +#endif + txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) + return txgbe_tx_done_cleanup_simple(txq, free_cnt); + + return txgbe_tx_done_cleanup_full(txq, free_cnt); +} + static void __rte_cold txgbe_tx_free_swring(struct txgbe_tx_queue *txq) { @@ -1993,6 +2165,9 @@ txgbe_set_tx_function(struct rte_eth_dev *dev, struct txgbe_tx_queue *txq) { /* Use a simple Tx queue (no offloads, no multi segs) if possible */ if (txq->offloads == 0 && +#ifdef RTE_LIB_SECURITY + !(txq->using_ipsec) && +#endif txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) { PMD_INIT_LOG(DEBUG, "Using simple tx code path"); dev->tx_pkt_burst = txgbe_xmit_pkts_simple; @@ -2047,6 +2222,10 @@ txgbe_get_tx_port_offloads(struct rte_eth_dev *dev) tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; +#ifdef RTE_LIB_SECURITY + if (dev->security_ctx) + tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY; +#endif return tx_offload_capa; } @@ -2145,6 +2324,10 @@ txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->offloads = offloads; txq->ops = &def_txq_ops; txq->tx_deferred_start = tx_conf->tx_deferred_start; +#ifdef RTE_LIB_SECURITY + txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads & + DEV_TX_OFFLOAD_SECURITY); +#endif /* Modification to set tail pointer for virtual function * if vf is detected. @@ -2501,6 +2684,79 @@ txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, return 0; } +uint32_t +txgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ +#define TXGBE_RXQ_SCAN_INTERVAL 4 + volatile struct txgbe_rx_desc *rxdp; + struct txgbe_rx_queue *rxq; + uint32_t desc = 0; + + rxq = dev->data->rx_queues[rx_queue_id]; + rxdp = &rxq->rx_ring[rxq->rx_tail]; + + while ((desc < rxq->nb_rx_desc) && + (rxdp->qw1.lo.status & + rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) { + desc += TXGBE_RXQ_SCAN_INTERVAL; + rxdp += TXGBE_RXQ_SCAN_INTERVAL; + if (rxq->rx_tail + desc >= rxq->nb_rx_desc) + rxdp = &(rxq->rx_ring[rxq->rx_tail + + desc - rxq->nb_rx_desc]); + } + + return desc; +} + +int +txgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) +{ + struct txgbe_rx_queue *rxq = rx_queue; + volatile uint32_t *status; + uint32_t nb_hold, desc; + + if (unlikely(offset >= rxq->nb_rx_desc)) + return -EINVAL; + + nb_hold = rxq->nb_rx_hold; + if (offset >= rxq->nb_rx_desc - nb_hold) + return RTE_ETH_RX_DESC_UNAVAIL; + + desc = rxq->rx_tail + offset; + if (desc >= rxq->nb_rx_desc) + desc -= rxq->nb_rx_desc; + + status = &rxq->rx_ring[desc].qw1.lo.status; + if (*status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)) + return RTE_ETH_RX_DESC_DONE; + + return RTE_ETH_RX_DESC_AVAIL; +} + +int +txgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) +{ + struct txgbe_tx_queue *txq = tx_queue; + volatile uint32_t *status; + uint32_t desc; + + if (unlikely(offset >= txq->nb_tx_desc)) + return -EINVAL; + + desc = txq->tx_tail + offset; + if (desc >= txq->nb_tx_desc) { + desc -= txq->nb_tx_desc; + if (desc >= txq->nb_tx_desc) + desc -= txq->nb_tx_desc; + } + + status = &txq->tx_ring[desc].dw3; + if (*status & rte_cpu_to_le_32(TXGBE_TXD_DD)) + return RTE_ETH_TX_DESC_DONE; + + return RTE_ETH_TX_DESC_FULL; +} + void __rte_cold txgbe_dev_clear_queues(struct rte_eth_dev *dev) { @@ -2548,137 +2804,1339 @@ txgbe_dev_free_queues(struct rte_eth_dev *dev) dev->data->nb_tx_queues = 0; } -static int __rte_cold -txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq) +/** + * Receive Side Scaling (RSS) + * + * Principles: + * The source and destination IP addresses of the IP header and the source + * and destination ports of TCP/UDP headers, if any, of received packets are + * hashed against a configurable random key to compute a 32-bit RSS hash result. + * The seven (7) LSBs of the 32-bit hash result are used as an index into a + * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit + * RSS output index which is used as the RX queue index where to store the + * received packets. + * The following output is supplied in the RX write-back descriptor: + * - 32-bit result of the Microsoft RSS hash function, + * - 4-bit RSS type field. + */ + +/* + * Used as the default key. + */ +static uint8_t rss_intel_key[40] = { + 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, + 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, + 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, + 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, + 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, +}; + +static void +txgbe_rss_disable(struct rte_eth_dev *dev) { - struct txgbe_rx_entry *rxe = rxq->sw_ring; - uint64_t dma_addr; - unsigned int i; + struct txgbe_hw *hw; - /* Initialize software ring entries */ - for (i = 0; i < rxq->nb_rx_desc; i++) { - volatile struct txgbe_rx_desc *rxd; - struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); + hw = TXGBE_DEV_HW(dev); + if (hw->mac.type == txgbe_mac_raptor_vf) + wr32m(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_RSSENA, 0); + else + wr32m(hw, TXGBE_RACTL, TXGBE_RACTL_RSSENA, 0); +} - if (mbuf == NULL) { - PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u", - (unsigned int)rxq->queue_id); - return -ENOMEM; +int +txgbe_dev_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + uint8_t *hash_key; + uint32_t mrqc; + uint32_t rss_key; + uint64_t rss_hf; + uint16_t i; + + if (!txgbe_rss_update_sp(hw->mac.type)) { + PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " + "NIC."); + return -ENOTSUP; + } + + hash_key = rss_conf->rss_key; + if (hash_key) { + /* Fill in RSS hash key */ + for (i = 0; i < 10; i++) { + rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF); + rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF); + rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF); + rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF); + wr32at(hw, TXGBE_REG_RSSKEY, i, rss_key); } + } - mbuf->data_off = RTE_PKTMBUF_HEADROOM; - mbuf->port = rxq->port_id; + /* Set configured hashing protocols */ + rss_hf = rss_conf->rss_hf & TXGBE_RSS_OFFLOAD_ALL; + if (hw->mac.type == txgbe_mac_raptor_vf) { + mrqc = rd32(hw, TXGBE_VFPLCFG); + mrqc &= ~TXGBE_VFPLCFG_RSSMASK; + if (rss_hf & ETH_RSS_IPV4) + mrqc |= TXGBE_VFPLCFG_RSSIPV4; + if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) + mrqc |= TXGBE_VFPLCFG_RSSIPV4TCP; + if (rss_hf & ETH_RSS_IPV6 || + rss_hf & ETH_RSS_IPV6_EX) + mrqc |= TXGBE_VFPLCFG_RSSIPV6; + if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP || + rss_hf & ETH_RSS_IPV6_TCP_EX) + mrqc |= TXGBE_VFPLCFG_RSSIPV6TCP; + if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) + mrqc |= TXGBE_VFPLCFG_RSSIPV4UDP; + if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP || + rss_hf & ETH_RSS_IPV6_UDP_EX) + mrqc |= TXGBE_VFPLCFG_RSSIPV6UDP; + + if (rss_hf) + mrqc |= TXGBE_VFPLCFG_RSSENA; + else + mrqc &= ~TXGBE_VFPLCFG_RSSENA; - dma_addr = - rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); - rxd = &rxq->rx_ring[i]; - TXGBE_RXD_HDRADDR(rxd, 0); - TXGBE_RXD_PKTADDR(rxd, dma_addr); - rxe[i].mbuf = mbuf; + if (dev->data->nb_rx_queues > 3) + mrqc |= TXGBE_VFPLCFG_RSSHASH(2); + else if (dev->data->nb_rx_queues > 1) + mrqc |= TXGBE_VFPLCFG_RSSHASH(1); + + wr32(hw, TXGBE_VFPLCFG, mrqc); + } else { + mrqc = rd32(hw, TXGBE_RACTL); + mrqc &= ~TXGBE_RACTL_RSSMASK; + if (rss_hf & ETH_RSS_IPV4) + mrqc |= TXGBE_RACTL_RSSIPV4; + if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) + mrqc |= TXGBE_RACTL_RSSIPV4TCP; + if (rss_hf & ETH_RSS_IPV6 || + rss_hf & ETH_RSS_IPV6_EX) + mrqc |= TXGBE_RACTL_RSSIPV6; + if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP || + rss_hf & ETH_RSS_IPV6_TCP_EX) + mrqc |= TXGBE_RACTL_RSSIPV6TCP; + if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) + mrqc |= TXGBE_RACTL_RSSIPV4UDP; + if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP || + rss_hf & ETH_RSS_IPV6_UDP_EX) + mrqc |= TXGBE_RACTL_RSSIPV6UDP; + + if (rss_hf) + mrqc |= TXGBE_RACTL_RSSENA; + else + mrqc &= ~TXGBE_RACTL_RSSENA; + + wr32(hw, TXGBE_RACTL, mrqc); } return 0; } -/** - * txgbe_get_rscctl_maxdesc - * - * @pool Memory pool of the Rx queue - */ -static inline uint32_t -txgbe_get_rscctl_maxdesc(struct rte_mempool *pool) +int +txgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) { - struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool); + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + uint8_t *hash_key; + uint32_t mrqc; + uint32_t rss_key; + uint64_t rss_hf; + uint16_t i; - uint16_t maxdesc = - RTE_IPV4_MAX_PKT_LEN / - (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM); + hash_key = rss_conf->rss_key; + if (hash_key) { + /* Return RSS hash key */ + for (i = 0; i < 10; i++) { + rss_key = rd32at(hw, TXGBE_REG_RSSKEY, i); + hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF); + hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF); + hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF); + hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF); + } + } - if (maxdesc >= 16) - return TXGBE_RXCFG_RSCMAX_16; - else if (maxdesc >= 8) - return TXGBE_RXCFG_RSCMAX_8; - else if (maxdesc >= 4) - return TXGBE_RXCFG_RSCMAX_4; - else - return TXGBE_RXCFG_RSCMAX_1; + rss_hf = 0; + if (hw->mac.type == txgbe_mac_raptor_vf) { + mrqc = rd32(hw, TXGBE_VFPLCFG); + if (mrqc & TXGBE_VFPLCFG_RSSIPV4) + rss_hf |= ETH_RSS_IPV4; + if (mrqc & TXGBE_VFPLCFG_RSSIPV4TCP) + rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; + if (mrqc & TXGBE_VFPLCFG_RSSIPV6) + rss_hf |= ETH_RSS_IPV6 | + ETH_RSS_IPV6_EX; + if (mrqc & TXGBE_VFPLCFG_RSSIPV6TCP) + rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_IPV6_TCP_EX; + if (mrqc & TXGBE_VFPLCFG_RSSIPV4UDP) + rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; + if (mrqc & TXGBE_VFPLCFG_RSSIPV6UDP) + rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_IPV6_UDP_EX; + if (!(mrqc & TXGBE_VFPLCFG_RSSENA)) + rss_hf = 0; + } else { + mrqc = rd32(hw, TXGBE_RACTL); + if (mrqc & TXGBE_RACTL_RSSIPV4) + rss_hf |= ETH_RSS_IPV4; + if (mrqc & TXGBE_RACTL_RSSIPV4TCP) + rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; + if (mrqc & TXGBE_RACTL_RSSIPV6) + rss_hf |= ETH_RSS_IPV6 | + ETH_RSS_IPV6_EX; + if (mrqc & TXGBE_RACTL_RSSIPV6TCP) + rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP | + ETH_RSS_IPV6_TCP_EX; + if (mrqc & TXGBE_RACTL_RSSIPV4UDP) + rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; + if (mrqc & TXGBE_RACTL_RSSIPV6UDP) + rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP | + ETH_RSS_IPV6_UDP_EX; + if (!(mrqc & TXGBE_RACTL_RSSENA)) + rss_hf = 0; + } + + rss_hf &= TXGBE_RSS_OFFLOAD_ALL; + + rss_conf->rss_hf = rss_hf; + return 0; } -/** - * txgbe_set_rsc - configure RSC related port HW registers - * - * Configures the port's RSC related registers. - * - * @dev port handle - * - * Returns 0 in case of success or a non-zero error code - */ -static int -txgbe_set_rsc(struct rte_eth_dev *dev) +static void +txgbe_rss_configure(struct rte_eth_dev *dev) { - struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; + struct rte_eth_rss_conf rss_conf; + struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); struct txgbe_hw *hw = TXGBE_DEV_HW(dev); - struct rte_eth_dev_info dev_info = { 0 }; - bool rsc_capable = false; + uint32_t reta; uint16_t i; - uint32_t rdrxctl; - uint32_t rfctl; + uint16_t j; - /* Sanity check */ - dev->dev_ops->dev_infos_get(dev, &dev_info); - if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) - rsc_capable = true; + PMD_INIT_FUNC_TRACE(); - if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { - PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't " - "support it"); - return -EINVAL; + /* + * Fill in redirection table + * The byte-swap is needed because NIC registers are in + * little-endian order. + */ + if (adapter->rss_reta_updated == 0) { + reta = 0; + for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) { + if (j == dev->data->nb_rx_queues) + j = 0; + reta = (reta >> 8) | LS32(j, 24, 0xFF); + if ((i & 3) == 3) + wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta); + } } + /* + * Configure the RSS key and the RSS protocols used to compute + * the RSS hash of input packets. + */ + rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; + if (rss_conf.rss_key == NULL) + rss_conf.rss_key = rss_intel_key; /* Default hash key */ + txgbe_dev_rss_hash_update(dev, &rss_conf); +} - /* RSC global configuration */ +#define NUM_VFTA_REGISTERS 128 +#define NIC_RX_BUFFER_SIZE 0x200 - if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) && - (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { - PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC " - "is disabled"); - return -EINVAL; +static void +txgbe_vmdq_dcb_configure(struct rte_eth_dev *dev) +{ + struct rte_eth_vmdq_dcb_conf *cfg; + struct txgbe_hw *hw; + enum rte_eth_nb_pools num_pools; + uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl; + uint16_t pbsize; + uint8_t nb_tcs; /* number of traffic classes */ + int i; + + PMD_INIT_FUNC_TRACE(); + hw = TXGBE_DEV_HW(dev); + cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; + num_pools = cfg->nb_queue_pools; + /* Check we have a valid number of pools */ + if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) { + txgbe_rss_disable(dev); + return; } + /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */ + nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools); - rfctl = rd32(hw, TXGBE_PSRCTL); - if (rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) - rfctl &= ~TXGBE_PSRCTL_RSCDIA; - else - rfctl |= TXGBE_PSRCTL_RSCDIA; - wr32(hw, TXGBE_PSRCTL, rfctl); + /* + * split rx buffer up into sections, each for 1 traffic class + */ + pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs); + for (i = 0; i < nb_tcs; i++) { + uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i)); + + rxpbsize &= (~(0x3FF << 10)); + /* clear 10 bits. */ + rxpbsize |= (pbsize << 10); /* set value */ + wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); + } + /* zero alloc all unused TCs */ + for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { + uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i)); - /* If LRO hasn't been requested - we are done here. */ - if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) - return 0; + rxpbsize &= (~(0x3FF << 10)); + /* clear 10 bits. */ + wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); + } - /* Set PSRCTL.RSCACK bit */ - rdrxctl = rd32(hw, TXGBE_PSRCTL); - rdrxctl |= TXGBE_PSRCTL_RSCACK; - wr32(hw, TXGBE_PSRCTL, rdrxctl); + if (num_pools == ETH_16_POOLS) { + mrqc = TXGBE_PORTCTL_NUMTC_8; + mrqc |= TXGBE_PORTCTL_NUMVT_16; + } else { + mrqc = TXGBE_PORTCTL_NUMTC_4; + mrqc |= TXGBE_PORTCTL_NUMVT_32; + } + wr32m(hw, TXGBE_PORTCTL, + TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK, mrqc); - /* Per-queue RSC configuration */ - for (i = 0; i < dev->data->nb_rx_queues; i++) { - struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; - uint32_t srrctl = - rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); - uint32_t psrtype = - rd32(hw, TXGBE_POOLRSS(rxq->reg_idx)); - uint32_t eitr = - rd32(hw, TXGBE_ITR(rxq->reg_idx)); + vt_ctl = TXGBE_POOLCTL_RPLEN; + if (cfg->enable_default_pool) + vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool); + else + vt_ctl |= TXGBE_POOLCTL_DEFDSA; + + wr32(hw, TXGBE_POOLCTL, vt_ctl); + queue_mapping = 0; + for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) /* - * txgbe PMD doesn't support header-split at the moment. + * mapping is done with 3 bits per priority, + * so shift by i*3 each time */ - srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK; - srrctl |= TXGBE_RXCFG_HDRLEN(128); + queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3)); - /* - * TODO: Consider setting the Receive Descriptor Minimum - * Threshold Size for an RSC case. This is not an obviously + wr32(hw, TXGBE_RPUP2TC, queue_mapping); + + wr32(hw, TXGBE_ARBRXCTL, TXGBE_ARBRXCTL_RRM); + + /* enable vlan filtering and allow all vlan tags through */ + vlanctrl = rd32(hw, TXGBE_VLANCTL); + vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ + wr32(hw, TXGBE_VLANCTL, vlanctrl); + + /* enable all vlan filters */ + for (i = 0; i < NUM_VFTA_REGISTERS; i++) + wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF); + + wr32(hw, TXGBE_POOLRXENA(0), + num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); + + wr32(hw, TXGBE_ETHADDRIDX, 0); + wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF); + wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF); + + /* set up filters for vlan tags as configured */ + for (i = 0; i < cfg->nb_pool_maps; i++) { + /* set vlan id in VF register and set the valid bit */ + wr32(hw, TXGBE_PSRVLANIDX, i); + wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA | + (cfg->pool_map[i].vlan_id & 0xFFF))); + + wr32(hw, TXGBE_PSRVLANPLM(0), cfg->pool_map[i].pools); + } +} + +/** + * txgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters + * @dev: pointer to eth_dev structure + * @dcb_config: pointer to txgbe_dcb_config structure + */ +static void +txgbe_dcb_tx_hw_config(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + uint32_t reg; + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + + PMD_INIT_FUNC_TRACE(); + + /* Disable the Tx desc arbiter */ + reg = rd32(hw, TXGBE_ARBTXCTL); + reg |= TXGBE_ARBTXCTL_DIA; + wr32(hw, TXGBE_ARBTXCTL, reg); + + /* Enable DCB for Tx with 8 TCs */ + reg = rd32(hw, TXGBE_PORTCTL); + reg &= TXGBE_PORTCTL_NUMTC_MASK; + reg |= TXGBE_PORTCTL_DCB; + if (dcb_config->num_tcs.pg_tcs == 8) + reg |= TXGBE_PORTCTL_NUMTC_8; + else + reg |= TXGBE_PORTCTL_NUMTC_4; + + wr32(hw, TXGBE_PORTCTL, reg); + + /* Enable the Tx desc arbiter */ + reg = rd32(hw, TXGBE_ARBTXCTL); + reg &= ~TXGBE_ARBTXCTL_DIA; + wr32(hw, TXGBE_ARBTXCTL, reg); +} + +/** + * txgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters + * @dev: pointer to rte_eth_dev structure + * @dcb_config: pointer to txgbe_dcb_config structure + */ +static void +txgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = + &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + + PMD_INIT_FUNC_TRACE(); + /*PF VF Transmit Enable*/ + wr32(hw, TXGBE_POOLTXENA(0), + vmdq_tx_conf->nb_queue_pools == + ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); + + /*Configure general DCB TX parameters*/ + txgbe_dcb_tx_hw_config(dev, dcb_config); +} + +static void +txgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = + &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; + struct txgbe_dcb_tc_config *tc; + uint8_t i, j; + + /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */ + if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) { + dcb_config->num_tcs.pg_tcs = ETH_8_TCS; + dcb_config->num_tcs.pfc_tcs = ETH_8_TCS; + } else { + dcb_config->num_tcs.pg_tcs = ETH_4_TCS; + dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; + } + + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; + } + + /* User Priority to Traffic Class mapping */ + for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { + j = vmdq_rx_conf->dcb_tc[i]; + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); + } +} + +static void +txgbe_dcb_vt_tx_config(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = + &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; + struct txgbe_dcb_tc_config *tc; + uint8_t i, j; + + /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */ + if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) { + dcb_config->num_tcs.pg_tcs = ETH_8_TCS; + dcb_config->num_tcs.pfc_tcs = ETH_8_TCS; + } else { + dcb_config->num_tcs.pg_tcs = ETH_4_TCS; + dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; + } + + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; + } + + /* User Priority to Traffic Class mapping */ + for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { + j = vmdq_tx_conf->dcb_tc[i]; + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); + } +} + +static void +txgbe_dcb_rx_config(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + struct rte_eth_dcb_rx_conf *rx_conf = + &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; + struct txgbe_dcb_tc_config *tc; + uint8_t i, j; + + dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs; + dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs; + + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; + } + + /* User Priority to Traffic Class mapping */ + for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { + j = rx_conf->dcb_tc[i]; + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); + } +} + +static void +txgbe_dcb_tx_config(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + struct rte_eth_dcb_tx_conf *tx_conf = + &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf; + struct txgbe_dcb_tc_config *tc; + uint8_t i, j; + + dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs; + dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs; + + /* Initialize User Priority to Traffic Class mapping */ + for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; + } + + /* User Priority to Traffic Class mapping */ + for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { + j = tx_conf->dcb_tc[i]; + tc = &dcb_config->tc_config[j]; + tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= + (uint8_t)(1 << i); + } +} + +/** + * txgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters + * @dev: pointer to eth_dev structure + * @dcb_config: pointer to txgbe_dcb_config structure + */ +static void +txgbe_dcb_rx_hw_config(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + uint32_t reg; + uint32_t vlanctrl; + uint8_t i; + uint32_t q; + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + + PMD_INIT_FUNC_TRACE(); + /* + * Disable the arbiter before changing parameters + * (always enable recycle mode; WSP) + */ + reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP | TXGBE_ARBRXCTL_DIA; + wr32(hw, TXGBE_ARBRXCTL, reg); + + reg = rd32(hw, TXGBE_PORTCTL); + reg &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); + if (dcb_config->num_tcs.pg_tcs == 4) { + reg |= TXGBE_PORTCTL_NUMTC_4; + if (dcb_config->vt_mode) + reg |= TXGBE_PORTCTL_NUMVT_32; + else + wr32(hw, TXGBE_POOLCTL, 0); + } + + if (dcb_config->num_tcs.pg_tcs == 8) { + reg |= TXGBE_PORTCTL_NUMTC_8; + if (dcb_config->vt_mode) + reg |= TXGBE_PORTCTL_NUMVT_16; + else + wr32(hw, TXGBE_POOLCTL, 0); + } + + wr32(hw, TXGBE_PORTCTL, reg); + + if (RTE_ETH_DEV_SRIOV(dev).active == 0) { + /* Disable drop for all queues in VMDQ mode*/ + for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) { + u32 val = 1 << (q % 32); + wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); + } + } else { + /* Enable drop for all queues in SRIOV mode */ + for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) { + u32 val = 1 << (q % 32); + wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); + } + } + + /* VLNCTL: enable vlan filtering and allow all vlan tags through */ + vlanctrl = rd32(hw, TXGBE_VLANCTL); + vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ + wr32(hw, TXGBE_VLANCTL, vlanctrl); + + /* VLANTBL - enable all vlan filters */ + for (i = 0; i < NUM_VFTA_REGISTERS; i++) + wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF); + + /* + * Configure Rx packet plane (recycle mode; WSP) and + * enable arbiter + */ + reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP; + wr32(hw, TXGBE_ARBRXCTL, reg); +} + +static void +txgbe_dcb_hw_arbite_rx_config(struct txgbe_hw *hw, uint16_t *refill, + uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) +{ + txgbe_dcb_config_rx_arbiter_raptor(hw, refill, max, bwg_id, + tsa, map); +} + +static void +txgbe_dcb_hw_arbite_tx_config(struct txgbe_hw *hw, uint16_t *refill, + uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) +{ + switch (hw->mac.type) { + case txgbe_mac_raptor: + txgbe_dcb_config_tx_desc_arbiter_raptor(hw, refill, + max, bwg_id, tsa); + txgbe_dcb_config_tx_data_arbiter_raptor(hw, refill, + max, bwg_id, tsa, map); + break; + default: + break; + } +} + +#define DCB_RX_CONFIG 1 +#define DCB_TX_CONFIG 1 +#define DCB_TX_PB 1024 +/** + * txgbe_dcb_hw_configure - Enable DCB and configure + * general DCB in VT mode and non-VT mode parameters + * @dev: pointer to rte_eth_dev structure + * @dcb_config: pointer to txgbe_dcb_config structure + */ +static int +txgbe_dcb_hw_configure(struct rte_eth_dev *dev, + struct txgbe_dcb_config *dcb_config) +{ + int ret = 0; + uint8_t i, pfc_en, nb_tcs; + uint16_t pbsize, rx_buffer_size; + uint8_t config_dcb_rx = 0; + uint8_t config_dcb_tx = 0; + uint8_t tsa[TXGBE_DCB_TC_MAX] = {0}; + uint8_t bwgid[TXGBE_DCB_TC_MAX] = {0}; + uint16_t refill[TXGBE_DCB_TC_MAX] = {0}; + uint16_t max[TXGBE_DCB_TC_MAX] = {0}; + uint8_t map[TXGBE_DCB_TC_MAX] = {0}; + struct txgbe_dcb_tc_config *tc; + uint32_t max_frame = dev->data->mtu + + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + struct txgbe_bw_conf *bw_conf = TXGBE_DEV_BW_CONF(dev); + + switch (dev->data->dev_conf.rxmode.mq_mode) { + case ETH_MQ_RX_VMDQ_DCB: + dcb_config->vt_mode = true; + config_dcb_rx = DCB_RX_CONFIG; + /* + * get dcb and VT rx configuration parameters + * from rte_eth_conf + */ + txgbe_vmdq_dcb_rx_config(dev, dcb_config); + /*Configure general VMDQ and DCB RX parameters*/ + txgbe_vmdq_dcb_configure(dev); + break; + case ETH_MQ_RX_DCB: + case ETH_MQ_RX_DCB_RSS: + dcb_config->vt_mode = false; + config_dcb_rx = DCB_RX_CONFIG; + /* Get dcb TX configuration parameters from rte_eth_conf */ + txgbe_dcb_rx_config(dev, dcb_config); + /*Configure general DCB RX parameters*/ + txgbe_dcb_rx_hw_config(dev, dcb_config); + break; + default: + PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration"); + break; + } + switch (dev->data->dev_conf.txmode.mq_mode) { + case ETH_MQ_TX_VMDQ_DCB: + dcb_config->vt_mode = true; + config_dcb_tx = DCB_TX_CONFIG; + /* get DCB and VT TX configuration parameters + * from rte_eth_conf + */ + txgbe_dcb_vt_tx_config(dev, dcb_config); + /* Configure general VMDQ and DCB TX parameters */ + txgbe_vmdq_dcb_hw_tx_config(dev, dcb_config); + break; + + case ETH_MQ_TX_DCB: + dcb_config->vt_mode = false; + config_dcb_tx = DCB_TX_CONFIG; + /* get DCB TX configuration parameters from rte_eth_conf */ + txgbe_dcb_tx_config(dev, dcb_config); + /* Configure general DCB TX parameters */ + txgbe_dcb_tx_hw_config(dev, dcb_config); + break; + default: + PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration"); + break; + } + + nb_tcs = dcb_config->num_tcs.pfc_tcs; + /* Unpack map */ + txgbe_dcb_unpack_map_cee(dcb_config, TXGBE_DCB_RX_CONFIG, map); + if (nb_tcs == ETH_4_TCS) { + /* Avoid un-configured priority mapping to TC0 */ + uint8_t j = 4; + uint8_t mask = 0xFF; + + for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++) + mask = (uint8_t)(mask & (~(1 << map[i]))); + for (i = 0; mask && (i < TXGBE_DCB_TC_MAX); i++) { + if ((mask & 0x1) && j < ETH_DCB_NUM_USER_PRIORITIES) + map[j++] = i; + mask >>= 1; + } + /* Re-configure 4 TCs BW */ + for (i = 0; i < nb_tcs; i++) { + tc = &dcb_config->tc_config[i]; + if (bw_conf->tc_num != nb_tcs) + tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = + (uint8_t)(100 / nb_tcs); + tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = + (uint8_t)(100 / nb_tcs); + } + for (; i < TXGBE_DCB_TC_MAX; i++) { + tc = &dcb_config->tc_config[i]; + tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 0; + tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 0; + } + } else { + /* Re-configure 8 TCs BW */ + for (i = 0; i < nb_tcs; i++) { + tc = &dcb_config->tc_config[i]; + if (bw_conf->tc_num != nb_tcs) + tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = + (uint8_t)(100 / nb_tcs + (i & 1)); + tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = + (uint8_t)(100 / nb_tcs + (i & 1)); + } + } + + rx_buffer_size = NIC_RX_BUFFER_SIZE; + + if (config_dcb_rx) { + /* Set RX buffer size */ + pbsize = (uint16_t)(rx_buffer_size / nb_tcs); + uint32_t rxpbsize = pbsize << 10; + + for (i = 0; i < nb_tcs; i++) + wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); + + /* zero alloc all unused TCs */ + for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) + wr32(hw, TXGBE_PBRXSIZE(i), 0); + } + if (config_dcb_tx) { + /* Only support an equally distributed + * Tx packet buffer strategy. + */ + uint32_t txpktsize = TXGBE_PBTXSIZE_MAX / nb_tcs; + uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - + TXGBE_TXPKT_SIZE_MAX; + + for (i = 0; i < nb_tcs; i++) { + wr32(hw, TXGBE_PBTXSIZE(i), txpktsize); + wr32(hw, TXGBE_PBTXDMATH(i), txpbthresh); + } + /* Clear unused TCs, if any, to zero buffer size*/ + for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { + wr32(hw, TXGBE_PBTXSIZE(i), 0); + wr32(hw, TXGBE_PBTXDMATH(i), 0); + } + } + + /*Calculates traffic class credits*/ + txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, + TXGBE_DCB_TX_CONFIG); + txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, + TXGBE_DCB_RX_CONFIG); + + if (config_dcb_rx) { + /* Unpack CEE standard containers */ + txgbe_dcb_unpack_refill_cee(dcb_config, + TXGBE_DCB_RX_CONFIG, refill); + txgbe_dcb_unpack_max_cee(dcb_config, max); + txgbe_dcb_unpack_bwgid_cee(dcb_config, + TXGBE_DCB_RX_CONFIG, bwgid); + txgbe_dcb_unpack_tsa_cee(dcb_config, + TXGBE_DCB_RX_CONFIG, tsa); + /* Configure PG(ETS) RX */ + txgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map); + } + + if (config_dcb_tx) { + /* Unpack CEE standard containers */ + txgbe_dcb_unpack_refill_cee(dcb_config, + TXGBE_DCB_TX_CONFIG, refill); + txgbe_dcb_unpack_max_cee(dcb_config, max); + txgbe_dcb_unpack_bwgid_cee(dcb_config, + TXGBE_DCB_TX_CONFIG, bwgid); + txgbe_dcb_unpack_tsa_cee(dcb_config, + TXGBE_DCB_TX_CONFIG, tsa); + /* Configure PG(ETS) TX */ + txgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map); + } + + /* Configure queue statistics registers */ + txgbe_dcb_config_tc_stats_raptor(hw, dcb_config); + + /* Check if the PFC is supported */ + if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) { + pbsize = (uint16_t)(rx_buffer_size / nb_tcs); + for (i = 0; i < nb_tcs; i++) { + /* If the TC count is 8, + * and the default high_water is 48, + * the low_water is 16 as default. + */ + hw->fc.high_water[i] = (pbsize * 3) / 4; + hw->fc.low_water[i] = pbsize / 4; + /* Enable pfc for this TC */ + tc = &dcb_config->tc_config[i]; + tc->pfc = txgbe_dcb_pfc_enabled; + } + txgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en); + if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS) + pfc_en &= 0x0F; + ret = txgbe_dcb_config_pfc(hw, pfc_en, map); + } + + return ret; +} + +void txgbe_configure_pb(struct rte_eth_dev *dev) +{ + struct rte_eth_conf *dev_conf = &dev->data->dev_conf; + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + + int hdrm; + int tc = dev_conf->rx_adv_conf.dcb_rx_conf.nb_tcs; + + /* Reserve 256KB(/512KB) rx buffer for fdir */ + hdrm = 256; /*KB*/ + + hw->mac.setup_pba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); +} + +void txgbe_configure_port(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + int i = 0; + uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, + 0x9100, 0x9200, + 0x0000, 0x0000, + 0x0000, 0x0000}; + + PMD_INIT_FUNC_TRACE(); + + /* default outer vlan tpid */ + wr32(hw, TXGBE_EXTAG, + TXGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) | + TXGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ)); + + /* default inner vlan tpid */ + wr32m(hw, TXGBE_VLANCTL, + TXGBE_VLANCTL_TPID_MASK, + TXGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN)); + wr32m(hw, TXGBE_DMATXCTRL, + TXGBE_DMATXCTRL_TPID_MASK, + TXGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN)); + + /* default vlan tpid filters */ + for (i = 0; i < 8; i++) { + wr32m(hw, TXGBE_TAGTPID(i / 2), + (i % 2 ? TXGBE_TAGTPID_MSB_MASK + : TXGBE_TAGTPID_LSB_MASK), + (i % 2 ? TXGBE_TAGTPID_MSB(tpids[i]) + : TXGBE_TAGTPID_LSB(tpids[i]))); + } + + /* default vxlan port */ + wr32(hw, TXGBE_VXLANPORT, 4789); +} + +/** + * txgbe_configure_dcb - Configure DCB Hardware + * @dev: pointer to rte_eth_dev + */ +void txgbe_configure_dcb(struct rte_eth_dev *dev) +{ + struct txgbe_dcb_config *dcb_cfg = TXGBE_DEV_DCB_CONFIG(dev); + struct rte_eth_conf *dev_conf = &dev->data->dev_conf; + + PMD_INIT_FUNC_TRACE(); + + /* check support mq_mode for DCB */ + if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB && + dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB && + dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS) + return; + + if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES) + return; + + /** Configure DCB hardware **/ + txgbe_dcb_hw_configure(dev, dcb_cfg); +} + +/* + * VMDq only support for 10 GbE NIC. + */ +static void +txgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev) +{ + struct rte_eth_vmdq_rx_conf *cfg; + struct txgbe_hw *hw; + enum rte_eth_nb_pools num_pools; + uint32_t mrqc, vt_ctl, vlanctrl; + uint32_t vmolr = 0; + int i; + + PMD_INIT_FUNC_TRACE(); + hw = TXGBE_DEV_HW(dev); + cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; + num_pools = cfg->nb_queue_pools; + + txgbe_rss_disable(dev); + + /* enable vmdq */ + mrqc = TXGBE_PORTCTL_NUMVT_64; + wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mrqc); + + /* turn on virtualisation and set the default pool */ + vt_ctl = TXGBE_POOLCTL_RPLEN; + if (cfg->enable_default_pool) + vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool); + else + vt_ctl |= TXGBE_POOLCTL_DEFDSA; + + wr32(hw, TXGBE_POOLCTL, vt_ctl); + + for (i = 0; i < (int)num_pools; i++) { + vmolr = txgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr); + wr32(hw, TXGBE_POOLETHCTL(i), vmolr); + } + + /* enable vlan filtering and allow all vlan tags through */ + vlanctrl = rd32(hw, TXGBE_VLANCTL); + vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ + wr32(hw, TXGBE_VLANCTL, vlanctrl); + + /* enable all vlan filters */ + for (i = 0; i < NUM_VFTA_REGISTERS; i++) + wr32(hw, TXGBE_VLANTBL(i), UINT32_MAX); + + /* pool enabling for receive - 64 */ + wr32(hw, TXGBE_POOLRXENA(0), UINT32_MAX); + if (num_pools == ETH_64_POOLS) + wr32(hw, TXGBE_POOLRXENA(1), UINT32_MAX); + + /* + * allow pools to read specific mac addresses + * In this case, all pools should be able to read from mac addr 0 + */ + wr32(hw, TXGBE_ETHADDRIDX, 0); + wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF); + wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF); + + /* set up filters for vlan tags as configured */ + for (i = 0; i < cfg->nb_pool_maps; i++) { + /* set vlan id in VF register and set the valid bit */ + wr32(hw, TXGBE_PSRVLANIDX, i); + wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA | + TXGBE_PSRVLAN_VID(cfg->pool_map[i].vlan_id))); + /* + * Put the allowed pools in VFB reg. As we only have 16 or 64 + * pools, we only need to use the first half of the register + * i.e. bits 0-31 + */ + if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0) + wr32(hw, TXGBE_PSRVLANPLM(0), + (cfg->pool_map[i].pools & UINT32_MAX)); + else + wr32(hw, TXGBE_PSRVLANPLM(1), + ((cfg->pool_map[i].pools >> 32) & UINT32_MAX)); + } + + /* Tx General Switch Control Enables VMDQ loopback */ + if (cfg->enable_loop_back) { + wr32(hw, TXGBE_PSRCTL, TXGBE_PSRCTL_LBENA); + for (i = 0; i < 64; i++) + wr32m(hw, TXGBE_POOLETHCTL(i), + TXGBE_POOLETHCTL_LLB, TXGBE_POOLETHCTL_LLB); + } + + txgbe_flush(hw); +} + +/* + * txgbe_vmdq_tx_hw_configure - Configure general VMDq TX parameters + * @hw: pointer to hardware structure + */ +static void +txgbe_vmdq_tx_hw_configure(struct txgbe_hw *hw) +{ + uint32_t reg; + uint32_t q; + + PMD_INIT_FUNC_TRACE(); + /*PF VF Transmit Enable*/ + wr32(hw, TXGBE_POOLTXENA(0), UINT32_MAX); + wr32(hw, TXGBE_POOLTXENA(1), UINT32_MAX); + + /* Disable the Tx desc arbiter */ + reg = rd32(hw, TXGBE_ARBTXCTL); + reg |= TXGBE_ARBTXCTL_DIA; + wr32(hw, TXGBE_ARBTXCTL, reg); + + wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, + TXGBE_PORTCTL_NUMVT_64); + + /* Disable drop for all queues */ + for (q = 0; q < 128; q++) { + u32 val = 1 << (q % 32); + wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); + } + + /* Enable the Tx desc arbiter */ + reg = rd32(hw, TXGBE_ARBTXCTL); + reg &= ~TXGBE_ARBTXCTL_DIA; + wr32(hw, TXGBE_ARBTXCTL, reg); + + txgbe_flush(hw); +} + +static int __rte_cold +txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq) +{ + struct txgbe_rx_entry *rxe = rxq->sw_ring; + uint64_t dma_addr; + unsigned int i; + + /* Initialize software ring entries */ + for (i = 0; i < rxq->nb_rx_desc; i++) { + volatile struct txgbe_rx_desc *rxd; + struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); + + if (mbuf == NULL) { + PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u", + (unsigned int)rxq->queue_id); + return -ENOMEM; + } + + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->port = rxq->port_id; + + dma_addr = + rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); + rxd = &rxq->rx_ring[i]; + TXGBE_RXD_HDRADDR(rxd, 0); + TXGBE_RXD_PKTADDR(rxd, dma_addr); + rxe[i].mbuf = mbuf; + } + + return 0; +} + +static int +txgbe_config_vf_rss(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw; + uint32_t mrqc; + + txgbe_rss_configure(dev); + + hw = TXGBE_DEV_HW(dev); + + /* enable VF RSS */ + mrqc = rd32(hw, TXGBE_PORTCTL); + mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); + switch (RTE_ETH_DEV_SRIOV(dev).active) { + case ETH_64_POOLS: + mrqc |= TXGBE_PORTCTL_NUMVT_64; + break; + + case ETH_32_POOLS: + mrqc |= TXGBE_PORTCTL_NUMVT_32; + break; + + default: + PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS"); + return -EINVAL; + } + + wr32(hw, TXGBE_PORTCTL, mrqc); + + return 0; +} + +static int +txgbe_config_vf_default(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + uint32_t mrqc; + + mrqc = rd32(hw, TXGBE_PORTCTL); + mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); + switch (RTE_ETH_DEV_SRIOV(dev).active) { + case ETH_64_POOLS: + mrqc |= TXGBE_PORTCTL_NUMVT_64; + break; + + case ETH_32_POOLS: + mrqc |= TXGBE_PORTCTL_NUMVT_32; + break; + + case ETH_16_POOLS: + mrqc |= TXGBE_PORTCTL_NUMVT_16; + break; + default: + PMD_INIT_LOG(ERR, + "invalid pool number in IOV mode"); + return 0; + } + + wr32(hw, TXGBE_PORTCTL, mrqc); + + return 0; +} + +static int +txgbe_dev_mq_rx_configure(struct rte_eth_dev *dev) +{ + if (RTE_ETH_DEV_SRIOV(dev).active == 0) { + /* + * SRIOV inactive scheme + * any DCB/RSS w/o VMDq multi-queue setting + */ + switch (dev->data->dev_conf.rxmode.mq_mode) { + case ETH_MQ_RX_RSS: + case ETH_MQ_RX_DCB_RSS: + case ETH_MQ_RX_VMDQ_RSS: + txgbe_rss_configure(dev); + break; + + case ETH_MQ_RX_VMDQ_DCB: + txgbe_vmdq_dcb_configure(dev); + break; + + case ETH_MQ_RX_VMDQ_ONLY: + txgbe_vmdq_rx_hw_configure(dev); + break; + + case ETH_MQ_RX_NONE: + default: + /* if mq_mode is none, disable rss mode.*/ + txgbe_rss_disable(dev); + break; + } + } else { + /* SRIOV active scheme + * Support RSS together with SRIOV. + */ + switch (dev->data->dev_conf.rxmode.mq_mode) { + case ETH_MQ_RX_RSS: + case ETH_MQ_RX_VMDQ_RSS: + txgbe_config_vf_rss(dev); + break; + case ETH_MQ_RX_VMDQ_DCB: + case ETH_MQ_RX_DCB: + /* In SRIOV, the configuration is the same as VMDq case */ + txgbe_vmdq_dcb_configure(dev); + break; + /* DCB/RSS together with SRIOV is not supported */ + case ETH_MQ_RX_VMDQ_DCB_RSS: + case ETH_MQ_RX_DCB_RSS: + PMD_INIT_LOG(ERR, + "Could not support DCB/RSS with VMDq & SRIOV"); + return -1; + default: + txgbe_config_vf_default(dev); + break; + } + } + + return 0; +} + +static int +txgbe_dev_mq_tx_configure(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + uint32_t mtqc; + uint32_t rttdcs; + + /* disable arbiter */ + rttdcs = rd32(hw, TXGBE_ARBTXCTL); + rttdcs |= TXGBE_ARBTXCTL_DIA; + wr32(hw, TXGBE_ARBTXCTL, rttdcs); + + if (RTE_ETH_DEV_SRIOV(dev).active == 0) { + /* + * SRIOV inactive scheme + * any DCB w/o VMDq multi-queue setting + */ + if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY) + txgbe_vmdq_tx_hw_configure(hw); + else + wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 0); + } else { + switch (RTE_ETH_DEV_SRIOV(dev).active) { + /* + * SRIOV active scheme + * FIXME if support DCB together with VMDq & SRIOV + */ + case ETH_64_POOLS: + mtqc = TXGBE_PORTCTL_NUMVT_64; + break; + case ETH_32_POOLS: + mtqc = TXGBE_PORTCTL_NUMVT_32; + break; + case ETH_16_POOLS: + mtqc = TXGBE_PORTCTL_NUMVT_16; + break; + default: + mtqc = 0; + PMD_INIT_LOG(ERR, "invalid pool number in IOV mode"); + } + wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mtqc); + } + + /* re-enable arbiter */ + rttdcs &= ~TXGBE_ARBTXCTL_DIA; + wr32(hw, TXGBE_ARBTXCTL, rttdcs); + + return 0; +} + +/** + * txgbe_get_rscctl_maxdesc + * + * @pool Memory pool of the Rx queue + */ +static inline uint32_t +txgbe_get_rscctl_maxdesc(struct rte_mempool *pool) +{ + struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool); + + uint16_t maxdesc = + RTE_IPV4_MAX_PKT_LEN / + (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM); + + if (maxdesc >= 16) + return TXGBE_RXCFG_RSCMAX_16; + else if (maxdesc >= 8) + return TXGBE_RXCFG_RSCMAX_8; + else if (maxdesc >= 4) + return TXGBE_RXCFG_RSCMAX_4; + else + return TXGBE_RXCFG_RSCMAX_1; +} + +/** + * txgbe_set_rsc - configure RSC related port HW registers + * + * Configures the port's RSC related registers. + * + * @dev port handle + * + * Returns 0 in case of success or a non-zero error code + */ +static int +txgbe_set_rsc(struct rte_eth_dev *dev) +{ + struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; + struct txgbe_hw *hw = TXGBE_DEV_HW(dev); + struct rte_eth_dev_info dev_info = { 0 }; + bool rsc_capable = false; + uint16_t i; + uint32_t rdrxctl; + uint32_t rfctl; + + /* Sanity check */ + dev->dev_ops->dev_infos_get(dev, &dev_info); + if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) + rsc_capable = true; + + if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { + PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't " + "support it"); + return -EINVAL; + } + + /* RSC global configuration */ + + if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) && + (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { + PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC " + "is disabled"); + return -EINVAL; + } + + rfctl = rd32(hw, TXGBE_PSRCTL); + if (rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) + rfctl &= ~TXGBE_PSRCTL_RSCDIA; + else + rfctl |= TXGBE_PSRCTL_RSCDIA; + wr32(hw, TXGBE_PSRCTL, rfctl); + + /* If LRO hasn't been requested - we are done here. */ + if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) + return 0; + + /* Set PSRCTL.RSCACK bit */ + rdrxctl = rd32(hw, TXGBE_PSRCTL); + rdrxctl |= TXGBE_PSRCTL_RSCACK; + wr32(hw, TXGBE_PSRCTL, rdrxctl); + + /* Per-queue RSC configuration */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; + uint32_t srrctl = + rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); + uint32_t psrtype = + rd32(hw, TXGBE_POOLRSS(rxq->reg_idx)); + uint32_t eitr = + rd32(hw, TXGBE_ITR(rxq->reg_idx)); + + /* + * txgbe PMD doesn't support header-split at the moment. + */ + srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK; + srrctl |= TXGBE_RXCFG_HDRLEN(128); + + /* + * TODO: Consider setting the Receive Descriptor Minimum + * Threshold Size for an RSC case. This is not an obviously * beneficiary option but the one worth considering... */ @@ -2725,6 +4183,7 @@ txgbe_set_rsc(struct rte_eth_dev *dev) void __rte_cold txgbe_set_rx_function(struct rte_eth_dev *dev) { + uint16_t i; struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); /* @@ -2785,6 +4244,15 @@ txgbe_set_rx_function(struct rte_eth_dev *dev) dev->rx_pkt_burst = txgbe_recv_pkts; } + +#ifdef RTE_LIB_SECURITY + for (i = 0; i < dev->data->nb_rx_queues; i++) { + struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; + + rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads & + DEV_RX_OFFLOAD_SECURITY); + } +#endif } /* @@ -2912,6 +4380,11 @@ txgbe_dev_rx_init(struct rte_eth_dev *dev) if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) dev->data->scattered_rx = 1; + /* + * Device configured with multiple RX queues. + */ + txgbe_dev_mq_rx_configure(dev); + /* * Setup the Checksum Register. * Disable Full-Packet Checksum which is mutually exclusive with RSS. @@ -2973,6 +4446,9 @@ txgbe_dev_tx_init(struct rte_eth_dev *dev) wr32(hw, TXGBE_TXRP(txq->reg_idx), 0); wr32(hw, TXGBE_TXWP(txq->reg_idx), 0); } + + /* Device configured with multiple TX queues. */ + txgbe_dev_mq_tx_configure(dev); } /* @@ -3047,6 +4523,19 @@ txgbe_dev_rxtx_start(struct rte_eth_dev *dev) dev->data->dev_conf.lpbk_mode) txgbe_setup_loopback_link_raptor(hw); +#ifdef RTE_LIB_SECURITY + if ((dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) || + (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_SECURITY)) { + ret = txgbe_crypto_enable_ipsec(dev); + if (ret != 0) { + PMD_DRV_LOG(ERR, + "txgbe_crypto_enable_ipsec fails with %d.", + ret); + return ret; + } + } +#endif + return 0; } @@ -3288,3 +4777,317 @@ txgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_deferred_start = txq->tx_deferred_start; } +/* + * [VF] Initializes Receive Unit. + */ +int __rte_cold +txgbevf_dev_rx_init(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw; + struct txgbe_rx_queue *rxq; + struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; + uint64_t bus_addr; + uint32_t srrctl, psrtype; + uint16_t buf_size; + uint16_t i; + int ret; + + PMD_INIT_FUNC_TRACE(); + hw = TXGBE_DEV_HW(dev); + + if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) { + PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " + "it should be power of 2"); + return -1; + } + + if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) { + PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " + "it should be equal to or less than %d", + hw->mac.max_rx_queues); + return -1; + } + + /* + * When the VF driver issues a TXGBE_VF_RESET request, the PF driver + * disables the VF receipt of packets if the PF MTU is > 1500. + * This is done to deal with limitations that imposes + * the PF and all VFs to share the same MTU. + * Then, the PF driver enables again the VF receipt of packet when + * the VF driver issues a TXGBE_VF_SET_LPE request. + * In the meantime, the VF device cannot be used, even if the VF driver + * and the Guest VM network stack are ready to accept packets with a + * size up to the PF MTU. + * As a work-around to this PF behaviour, force the call to + * txgbevf_rlpml_set_vf even if jumbo frames are not used. This way, + * VF packets received can work in all cases. + */ + if (txgbevf_rlpml_set_vf(hw, + (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len)) { + PMD_INIT_LOG(ERR, "Set max packet length to %d failed.", + dev->data->dev_conf.rxmode.max_rx_pkt_len); + return -EINVAL; + } + + /* + * Assume no header split and no VLAN strip support + * on any Rx queue first . + */ + rxmode->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; + + /* Set PSR type for VF RSS according to max Rx queue */ + psrtype = TXGBE_VFPLCFG_PSRL4HDR | + TXGBE_VFPLCFG_PSRL4HDR | + TXGBE_VFPLCFG_PSRL2HDR | + TXGBE_VFPLCFG_PSRTUNHDR | + TXGBE_VFPLCFG_PSRTUNMAC; + wr32(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_PSR(psrtype)); + + /* Setup RX queues */ + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + + /* Allocate buffers for descriptor rings */ + ret = txgbe_alloc_rx_queue_mbufs(rxq); + if (ret) + return ret; + + /* Setup the Base and Length of the Rx Descriptor Rings */ + bus_addr = rxq->rx_ring_phys_addr; + + wr32(hw, TXGBE_RXBAL(i), + (uint32_t)(bus_addr & BIT_MASK32)); + wr32(hw, TXGBE_RXBAH(i), + (uint32_t)(bus_addr >> 32)); + wr32(hw, TXGBE_RXRP(i), 0); + wr32(hw, TXGBE_RXWP(i), 0); + + /* Configure the RXCFG register */ + srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); + + /* Set if packets are dropped when no descriptors available */ + if (rxq->drop_en) + srrctl |= TXGBE_RXCFG_DROP; + + /* + * Configure the RX buffer size in the PKTLEN field of + * the RXCFG register of the queue. + * The value is in 1 KB resolution. Valid values can be from + * 1 KB to 16 KB. + */ + buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - + RTE_PKTMBUF_HEADROOM); + buf_size = ROUND_UP(buf_size, 1 << 10); + srrctl |= TXGBE_RXCFG_PKTLEN(buf_size); + + /* + * VF modification to write virtual function RXCFG register + */ + wr32(hw, TXGBE_RXCFG(i), srrctl); + + if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER || + /* It adds dual VLAN length for supporting dual VLAN */ + (rxmode->max_rx_pkt_len + + 2 * TXGBE_VLAN_TAG_SIZE) > buf_size) { + if (!dev->data->scattered_rx) + PMD_INIT_LOG(DEBUG, "forcing scatter mode"); + dev->data->scattered_rx = 1; + } + + if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) + rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; + } + + /* + * Device configured with multiple RX queues. + */ + txgbe_dev_mq_rx_configure(dev); + + txgbe_set_rx_function(dev); + + return 0; +} + +/* + * [VF] Initializes Transmit Unit. + */ +void __rte_cold +txgbevf_dev_tx_init(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw; + struct txgbe_tx_queue *txq; + uint64_t bus_addr; + uint16_t i; + + PMD_INIT_FUNC_TRACE(); + hw = TXGBE_DEV_HW(dev); + + /* Setup the Base and Length of the Tx Descriptor Rings */ + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + bus_addr = txq->tx_ring_phys_addr; + wr32(hw, TXGBE_TXBAL(i), + (uint32_t)(bus_addr & BIT_MASK32)); + wr32(hw, TXGBE_TXBAH(i), + (uint32_t)(bus_addr >> 32)); + wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_BUFLEN_MASK, + TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); + /* Setup the HW Tx Head and TX Tail descriptor pointers */ + wr32(hw, TXGBE_TXRP(i), 0); + wr32(hw, TXGBE_TXWP(i), 0); + } +} + +/* + * [VF] Start Transmit and Receive Units. + */ +void __rte_cold +txgbevf_dev_rxtx_start(struct rte_eth_dev *dev) +{ + struct txgbe_hw *hw; + struct txgbe_tx_queue *txq; + struct txgbe_rx_queue *rxq; + uint32_t txdctl; + uint32_t rxdctl; + uint16_t i; + int poll_ms; + + PMD_INIT_FUNC_TRACE(); + hw = TXGBE_DEV_HW(dev); + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + /* Setup Transmit Threshold Registers */ + wr32m(hw, TXGBE_TXCFG(txq->reg_idx), + TXGBE_TXCFG_HTHRESH_MASK | + TXGBE_TXCFG_WTHRESH_MASK, + TXGBE_TXCFG_HTHRESH(txq->hthresh) | + TXGBE_TXCFG_WTHRESH(txq->wthresh)); + } + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA); + + poll_ms = 10; + /* Wait until TX Enable ready */ + do { + rte_delay_ms(1); + txdctl = rd32(hw, TXGBE_TXCFG(i)); + } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); + } + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + + wr32m(hw, TXGBE_RXCFG(i), TXGBE_RXCFG_ENA, TXGBE_RXCFG_ENA); + + /* Wait until RX Enable ready */ + poll_ms = 10; + do { + rte_delay_ms(1); + rxdctl = rd32(hw, TXGBE_RXCFG(i)); + } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA)); + if (!poll_ms) + PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); + rte_wmb(); + wr32(hw, TXGBE_RXWP(i), rxq->nb_rx_desc - 1); + } +} + +int +txgbe_rss_conf_init(struct txgbe_rte_flow_rss_conf *out, + const struct rte_flow_action_rss *in) +{ + if (in->key_len > RTE_DIM(out->key) || + in->queue_num > RTE_DIM(out->queue)) + return -EINVAL; + out->conf = (struct rte_flow_action_rss){ + .func = in->func, + .level = in->level, + .types = in->types, + .key_len = in->key_len, + .queue_num = in->queue_num, + .key = memcpy(out->key, in->key, in->key_len), + .queue = memcpy(out->queue, in->queue, + sizeof(*in->queue) * in->queue_num), + }; + return 0; +} + +int +txgbe_action_rss_same(const struct rte_flow_action_rss *comp, + const struct rte_flow_action_rss *with) +{ + return (comp->func == with->func && + comp->level == with->level && + comp->types == with->types && + comp->key_len == with->key_len && + comp->queue_num == with->queue_num && + !memcmp(comp->key, with->key, with->key_len) && + !memcmp(comp->queue, with->queue, + sizeof(*with->queue) * with->queue_num)); +} + +int +txgbe_config_rss_filter(struct rte_eth_dev *dev, + struct txgbe_rte_flow_rss_conf *conf, bool add) +{ + struct txgbe_hw *hw; + uint32_t reta; + uint16_t i; + uint16_t j; + struct rte_eth_rss_conf rss_conf = { + .rss_key = conf->conf.key_len ? + (void *)(uintptr_t)conf->conf.key : NULL, + .rss_key_len = conf->conf.key_len, + .rss_hf = conf->conf.types, + }; + struct txgbe_filter_info *filter_info = TXGBE_DEV_FILTER(dev); + + PMD_INIT_FUNC_TRACE(); + hw = TXGBE_DEV_HW(dev); + + if (!add) { + if (txgbe_action_rss_same(&filter_info->rss_info.conf, + &conf->conf)) { + txgbe_rss_disable(dev); + memset(&filter_info->rss_info, 0, + sizeof(struct txgbe_rte_flow_rss_conf)); + return 0; + } + return -EINVAL; + } + + if (filter_info->rss_info.conf.queue_num) + return -EINVAL; + /* Fill in redirection table + * The byte-swap is needed because NIC registers are in + * little-endian order. + */ + reta = 0; + for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) { + if (j == conf->conf.queue_num) + j = 0; + reta = (reta >> 8) | LS32(conf->conf.queue[j], 24, 0xFF); + if ((i & 3) == 3) + wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta); + } + + /* Configure the RSS key and the RSS protocols used to compute + * the RSS hash of input packets. + */ + if ((rss_conf.rss_hf & TXGBE_RSS_OFFLOAD_ALL) == 0) { + txgbe_rss_disable(dev); + return 0; + } + if (rss_conf.rss_key == NULL) + rss_conf.rss_key = rss_intel_key; /* Default hash key */ + txgbe_dev_rss_hash_update(dev, &rss_conf); + + if (txgbe_rss_conf_init(&filter_info->rss_info, &conf->conf)) + return -EINVAL; + + return 0; +} +