X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fhns3%2Fhns3_rxtx.c;h=70de0d2b5863847d8336f73a6adaaa7dc86c18b5;hb=364067380ecf7b8c1b30178b4152f244b1c1b03b;hp=a76b141891cbe2e295e1cb9a9220995b5a46481c;hpb=4801f0403b58dcead23b778085f6180b7789bfac;p=dpdk.git

diff --git a/drivers/net/hns3/hns3_rxtx.c b/drivers/net/hns3/hns3_rxtx.c
index a76b141891..70de0d2b58 100644
--- a/drivers/net/hns3/hns3_rxtx.c
+++ b/drivers/net/hns3/hns3_rxtx.c
@@ -20,6 +20,7 @@
 #include "hns3_rxtx.h"
 #include "hns3_regs.h"
 #include "hns3_logs.h"
+#include "hns3_mp.h"
 
 #define HNS3_CFG_DESC_NUM(num)	((num) / 8 - 1)
 #define HNS3_RX_RING_PREFETCTH_MASK	3
@@ -107,8 +108,8 @@ hns3_tx_queue_release(void *queue)
 	}
 }
 
-void
-hns3_dev_rx_queue_release(void *queue)
+static void
+hns3_rx_queue_release_lock(void *queue)
 {
 	struct hns3_rx_queue *rxq = queue;
 	struct hns3_adapter *hns;
@@ -123,7 +124,13 @@ hns3_dev_rx_queue_release(void *queue)
 }
 
 void
-hns3_dev_tx_queue_release(void *queue)
+hns3_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+	hns3_rx_queue_release_lock(dev->data->rx_queues[queue_id]);
+}
+
+static void
+hns3_tx_queue_release_lock(void *queue)
 {
 	struct hns3_tx_queue *txq = queue;
 	struct hns3_adapter *hns;
@@ -137,6 +144,12 @@ hns3_dev_tx_queue_release(void *queue)
 	rte_spinlock_unlock(&hns->hw.lock);
 }
 
+void
+hns3_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+	hns3_tx_queue_release_lock(dev->data->tx_queues[queue_id]);
+}
+
 static void
 hns3_fake_rx_queue_release(struct hns3_rx_queue *queue)
 {
@@ -380,7 +393,7 @@ hns3_enable_all_queues(struct hns3_hw *hw, bool en)
 	int i;
 
 	for (i = 0; i < hw->cfg_max_queues; i++) {
-		if (hns3_dev_indep_txrx_supported(hw)) {
+		if (hns3_dev_get_support(hw, INDEP_TXRX)) {
 			rxq = i < nb_rx_q ? hw->data->rx_queues[i] : NULL;
 			txq = i < nb_tx_q ? hw->data->tx_queues[i] : NULL;
 
@@ -425,7 +438,7 @@ hns3_enable_txq(struct hns3_tx_queue *txq, bool en)
 	struct hns3_hw *hw = &txq->hns->hw;
 	uint32_t reg;
 
-	if (hns3_dev_indep_txrx_supported(hw)) {
+	if (hns3_dev_get_support(hw, INDEP_TXRX)) {
 		reg = hns3_read_dev(txq, HNS3_RING_TX_EN_REG);
 		if (en)
 			reg |= BIT(HNS3_RING_EN_B);
@@ -442,7 +455,7 @@ hns3_enable_rxq(struct hns3_rx_queue *rxq, bool en)
 	struct hns3_hw *hw = &rxq->hns->hw;
 	uint32_t reg;
 
-	if (hns3_dev_indep_txrx_supported(hw)) {
+	if (hns3_dev_get_support(hw, INDEP_TXRX)) {
 		reg = hns3_read_dev(rxq, HNS3_RING_RX_EN_REG);
 		if (en)
 			reg |= BIT(HNS3_RING_EN_B);
@@ -696,7 +709,7 @@ hns3_reset_rcb_cmd(struct hns3_hw *hw, uint8_t *reset_status)
 
 	hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_CFG_RST_TRIGGER, false);
 	req = (struct hns3_reset_cmd *)desc.data;
-	hns3_set_bit(req->mac_func_reset, HNS3_CFG_RESET_RCB_B, 1);
+	hns3_set_bit(req->fun_reset_rcb, HNS3_CFG_RESET_RCB_B, 1);
 
 	/*
 	 * The start qid should be the global qid of the first tqp of the
@@ -1535,7 +1548,7 @@ hns3_fake_rx_queue_config(struct hns3_hw *hw, uint16_t nb_queues)
 		/* re-configure */
 		rxq = hw->fkq_data.rx_queues;
 		for (i = nb_queues; i < old_nb_queues; i++)
-			hns3_dev_rx_queue_release(rxq[i]);
+			hns3_rx_queue_release_lock(rxq[i]);
 
 		rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 				  RTE_CACHE_LINE_SIZE);
@@ -1550,7 +1563,7 @@ hns3_fake_rx_queue_config(struct hns3_hw *hw, uint16_t nb_queues)
 	} else if (hw->fkq_data.rx_queues != NULL && nb_queues == 0) {
 		rxq = hw->fkq_data.rx_queues;
 		for (i = nb_queues; i < old_nb_queues; i++)
-			hns3_dev_rx_queue_release(rxq[i]);
+			hns3_rx_queue_release_lock(rxq[i]);
 
 		rte_free(hw->fkq_data.rx_queues);
 		hw->fkq_data.rx_queues = NULL;
@@ -1582,7 +1595,7 @@ hns3_fake_tx_queue_config(struct hns3_hw *hw, uint16_t nb_queues)
 		/* re-configure */
 		txq = hw->fkq_data.tx_queues;
 		for (i = nb_queues; i < old_nb_queues; i++)
-			hns3_dev_tx_queue_release(txq[i]);
+			hns3_tx_queue_release_lock(txq[i]);
 		txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
 				  RTE_CACHE_LINE_SIZE);
 		if (txq == NULL)
@@ -1596,7 +1609,7 @@ hns3_fake_tx_queue_config(struct hns3_hw *hw, uint16_t nb_queues)
 	} else if (hw->fkq_data.tx_queues != NULL && nb_queues == 0) {
 		txq = hw->fkq_data.tx_queues;
 		for (i = nb_queues; i < old_nb_queues; i++)
-			hns3_dev_tx_queue_release(txq[i]);
+			hns3_tx_queue_release_lock(txq[i]);
 
 		rte_free(hw->fkq_data.tx_queues);
 		hw->fkq_data.tx_queues = NULL;
@@ -1617,6 +1630,9 @@ hns3_set_fake_rx_or_tx_queues(struct rte_eth_dev *dev, uint16_t nb_rx_q,
 	uint16_t q;
 	int ret;
 
+	if (hns3_dev_get_support(hw, INDEP_TXRX))
+		return 0;
+
 	/* Setup new number of fake RX/TX queues and reconfigure device. */
 	rx_need_add_nb_q = hw->cfg_max_queues - nb_rx_q;
 	tx_need_add_nb_q = hw->cfg_max_queues - nb_tx_q;
@@ -1858,7 +1874,7 @@ hns3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
 		conf->rx_free_thresh : HNS3_DEFAULT_RX_FREE_THRESH;
 
 	rxq->rx_deferred_start = conf->rx_deferred_start;
-	if (rxq->rx_deferred_start && !hns3_dev_indep_txrx_supported(hw)) {
+	if (rxq->rx_deferred_start && !hns3_dev_get_support(hw, INDEP_TXRX)) {
 		hns3_warn(hw, "deferred start is not supported.");
 		rxq->rx_deferred_start = false;
 	}
@@ -1894,7 +1910,7 @@ hns3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
 				       HNS3_PORT_BASE_VLAN_ENABLE;
 	else
 		rxq->pvid_sw_discard_en = false;
-	rxq->ptype_en = hns3_dev_rxd_adv_layout_supported(hw) ? true : false;
+	rxq->ptype_en = hns3_dev_get_support(hw, RXD_ADV_LAYOUT) ? true : false;
 	rxq->configured = true;
 	rxq->io_base = (void *)((char *)hw->io_base + HNS3_TQP_REG_OFFSET +
 				idx * HNS3_TQP_REG_SIZE);
@@ -2022,7 +2038,7 @@ hns3_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 	    dev->rx_pkt_burst == hns3_recv_scattered_pkts ||
 	    dev->rx_pkt_burst == hns3_recv_pkts_vec ||
 	    dev->rx_pkt_burst == hns3_recv_pkts_vec_sve) {
-		if (hns3_dev_rxd_adv_layout_supported(hw))
+		if (hns3_dev_get_support(hw, RXD_ADV_LAYOUT))
 			return adv_layout_ptypes;
 		else
 			return ptypes;
@@ -2808,7 +2824,7 @@ hns3_get_default_vec_support(void)
 static bool
 hns3_get_sve_support(void)
 {
-#if defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_SVE)
+#if defined(RTE_HAS_SVE_ACLE)
 	if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_256)
 		return false;
 	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SVE))
@@ -2892,6 +2908,69 @@ hns3_tx_queue_conf_check(struct hns3_hw *hw, const struct rte_eth_txconf *conf,
 	return 0;
 }
 
+static void *
+hns3_tx_push_get_queue_tail_reg(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+#define HNS3_TX_PUSH_TQP_REGION_SIZE		0x10000
+#define HNS3_TX_PUSH_QUICK_DOORBELL_OFFSET	64
+#define HNS3_TX_PUSH_PCI_BAR_INDEX		4
+
+	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
+	uint8_t bar_id = HNS3_TX_PUSH_PCI_BAR_INDEX;
+
+	/*
+	 * If device support Tx push then its PCIe bar45 must exist, and DPDK
+	 * framework will mmap the bar45 default in PCI probe stage.
+	 *
+	 * In the bar45, the first half is for RoCE (RDMA over Converged
+	 * Ethernet), and the second half is for NIC, every TQP occupy 64KB.
+	 *
+	 * The quick doorbell located at 64B offset in the TQP region.
+	 */
+	return (char *)pci_dev->mem_resource[bar_id].addr +
+			(pci_dev->mem_resource[bar_id].len >> 1) +
+			HNS3_TX_PUSH_TQP_REGION_SIZE * queue_id +
+			HNS3_TX_PUSH_QUICK_DOORBELL_OFFSET;
+}
+
+void
+hns3_tx_push_init(struct rte_eth_dev *dev)
+{
+	struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	volatile uint32_t *reg;
+	uint32_t val;
+
+	if (!hns3_dev_get_support(hw, TX_PUSH))
+		return;
+
+	reg = (volatile uint32_t *)hns3_tx_push_get_queue_tail_reg(dev, 0);
+	/*
+	 * Because the size of bar45 is about 8GB size, it may take a long time
+	 * to do the page fault in Tx process when work with vfio-pci, so use
+	 * one read operation to make kernel setup page table mapping for bar45
+	 * in the init stage.
+	 * Note: the bar45 is readable but the result is all 1.
+	 */
+	val = *reg;
+	RTE_SET_USED(val);
+}
+
+static void
+hns3_tx_push_queue_init(struct rte_eth_dev *dev,
+			uint16_t queue_id,
+			struct hns3_tx_queue *txq)
+{
+	struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	if (!hns3_dev_get_support(hw, TX_PUSH)) {
+		txq->tx_push_enable = false;
+		return;
+	}
+
+	txq->io_tail_reg = (volatile void *)hns3_tx_push_get_queue_tail_reg(dev,
+						queue_id);
+	txq->tx_push_enable = true;
+}
+
 int
 hns3_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
 		    unsigned int socket_id, const struct rte_eth_txconf *conf)
@@ -2927,7 +3006,7 @@ hns3_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
 	}
 
 	txq->tx_deferred_start = conf->tx_deferred_start;
-	if (txq->tx_deferred_start && !hns3_dev_indep_txrx_supported(hw)) {
+	if (txq->tx_deferred_start && !hns3_dev_get_support(hw, INDEP_TXRX)) {
 		hns3_warn(hw, "deferred start is not supported.");
 		txq->tx_deferred_start = false;
 	}
@@ -2983,6 +3062,12 @@ hns3_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
 	memset(&txq->basic_stats, 0, sizeof(struct hns3_tx_basic_stats));
 	memset(&txq->dfx_stats, 0, sizeof(struct hns3_tx_dfx_stats));
 
+	/*
+	 * Call hns3_tx_push_queue_init after assigned io_tail_reg field because
+	 * it may overwrite the io_tail_reg field.
+	 */
+	hns3_tx_push_queue_init(dev, idx, txq);
+
 	rte_spinlock_lock(&hw->lock);
 	dev->data->tx_queues[idx] = txq;
 	rte_spinlock_unlock(&hw->lock);
@@ -4029,7 +4114,7 @@ hns3_xmit_pkts_simple(void *tx_queue,
 	hns3_tx_fill_hw_ring(txq, tx_pkts + nb_tx, nb_pkts - nb_tx);
 	txq->next_to_use += nb_pkts - nb_tx;
 
-	hns3_write_reg_opt(txq->io_tail_reg, nb_pkts);
+	hns3_write_txq_tail_reg(txq, nb_pkts);
 
 	return nb_pkts;
 }
@@ -4146,7 +4231,7 @@ hns3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 end_of_tx:
 
 	if (likely(nb_tx))
-		hns3_write_reg_opt(txq->io_tail_reg, nb_hold);
+		hns3_write_txq_tail_reg(txq, nb_hold);
 
 	return nb_tx;
 }
@@ -4203,7 +4288,7 @@ hns3_tx_check_simple_support(struct rte_eth_dev *dev)
 	uint64_t offloads = dev->data->dev_conf.txmode.offloads;
 
 	struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
-	if (hns3_dev_ptp_supported(hw))
+	if (hns3_dev_get_support(hw, PTP))
 		return false;
 
 	return (offloads == (offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE));
@@ -4213,6 +4298,7 @@ static bool
 hns3_get_tx_prep_needed(struct rte_eth_dev *dev)
 {
 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
+	RTE_SET_USED(dev);
 	/* always perform tx_prepare when debug */
 	return true;
 #else
@@ -4236,7 +4322,7 @@ hns3_get_tx_prep_needed(struct rte_eth_dev *dev)
 #endif
 }
 
-static eth_tx_burst_t
+eth_tx_burst_t
 hns3_get_tx_function(struct rte_eth_dev *dev, eth_tx_prep_t *prep)
 {
 	struct hns3_adapter *hns = dev->data->dev_private;
@@ -4273,7 +4359,7 @@ hns3_get_tx_function(struct rte_eth_dev *dev, eth_tx_prep_t *prep)
 	return hns3_xmit_pkts;
 }
 
-static uint16_t
+uint16_t
 hns3_dummy_rxtx_burst(void *dpdk_txq __rte_unused,
 		      struct rte_mbuf **pkts __rte_unused,
 		      uint16_t pkts_n __rte_unused)
@@ -4299,6 +4385,7 @@ hns3_trace_rxtx_function(struct rte_eth_dev *dev)
 
 void hns3_set_rxtx_function(struct rte_eth_dev *eth_dev)
 {
+	struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 	struct hns3_adapter *hns = eth_dev->data->dev_private;
 	eth_tx_prep_t prep = NULL;
 
@@ -4306,14 +4393,16 @@ void hns3_set_rxtx_function(struct rte_eth_dev *eth_dev)
 	    __atomic_load_n(&hns->hw.reset.resetting, __ATOMIC_RELAXED) == 0) {
 		eth_dev->rx_pkt_burst = hns3_get_rx_function(eth_dev);
 		eth_dev->rx_descriptor_status = hns3_dev_rx_descriptor_status;
-		eth_dev->tx_pkt_burst = hns3_get_tx_function(eth_dev, &prep);
+		eth_dev->tx_pkt_burst = hw->set_link_down ?
+					hns3_dummy_rxtx_burst :
+					hns3_get_tx_function(eth_dev, &prep);
 		eth_dev->tx_pkt_prepare = prep;
 		eth_dev->tx_descriptor_status = hns3_dev_tx_descriptor_status;
 		hns3_trace_rxtx_function(eth_dev);
 	} else {
 		eth_dev->rx_pkt_burst = hns3_dummy_rxtx_burst;
 		eth_dev->tx_pkt_burst = hns3_dummy_rxtx_burst;
-		eth_dev->tx_pkt_prepare = hns3_dummy_rxtx_burst;
+		eth_dev->tx_pkt_prepare = NULL;
 	}
 }
 
@@ -4360,7 +4449,7 @@ hns3_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw);
 	int ret;
 
-	if (!hns3_dev_indep_txrx_supported(hw))
+	if (!hns3_dev_get_support(hw, INDEP_TXRX))
 		return -ENOTSUP;
 
 	rte_spinlock_lock(&hw->lock);
@@ -4406,7 +4495,7 @@ hns3_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct hns3_rx_queue *rxq = dev->data->rx_queues[rx_queue_id];
 
-	if (!hns3_dev_indep_txrx_supported(hw))
+	if (!hns3_dev_get_support(hw, INDEP_TXRX))
 		return -ENOTSUP;
 
 	rte_spinlock_lock(&hw->lock);
@@ -4428,7 +4517,7 @@ hns3_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct hns3_tx_queue *txq = dev->data->tx_queues[tx_queue_id];
 	int ret;
 
-	if (!hns3_dev_indep_txrx_supported(hw))
+	if (!hns3_dev_get_support(hw, INDEP_TXRX))
 		return -ENOTSUP;
 
 	rte_spinlock_lock(&hw->lock);
@@ -4454,7 +4543,7 @@ hns3_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct hns3_tx_queue *txq = dev->data->tx_queues[tx_queue_id];
 
-	if (!hns3_dev_indep_txrx_supported(hw))
+	if (!hns3_dev_get_support(hw, INDEP_TXRX))
 		return -ENOTSUP;
 
 	rte_spinlock_lock(&hw->lock);
@@ -4627,6 +4716,28 @@ hns3_enable_rxd_adv_layout(struct hns3_hw *hw)
 	 * If the hardware support rxd advanced layout, then driver enable it
 	 * default.
 	 */
-	if (hns3_dev_rxd_adv_layout_supported(hw))
+	if (hns3_dev_get_support(hw, RXD_ADV_LAYOUT))
 		hns3_write_dev(hw, HNS3_RXD_ADV_LAYOUT_EN_REG, 1);
 }
+
+void
+hns3_stop_tx_datapath(struct rte_eth_dev *dev)
+{
+	dev->tx_pkt_burst = hns3_dummy_rxtx_burst;
+	dev->tx_pkt_prepare = NULL;
+	rte_wmb();
+	/* Disable tx datapath on secondary process. */
+	hns3_mp_req_stop_tx(dev);
+	/* Prevent crashes when queues are still in use. */
+	rte_delay_ms(dev->data->nb_tx_queues);
+}
+
+void
+hns3_start_tx_datapath(struct rte_eth_dev *dev)
+{
+	eth_tx_prep_t prep = NULL;
+
+	dev->tx_pkt_burst = hns3_get_tx_function(dev, &prep);
+	dev->tx_pkt_prepare = prep;
+	hns3_mp_req_start_tx(dev);
+}