+#if defined(RTE_ARCH_ARM64)
+ if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)
+ return false;
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON))
+ return true;
+#endif
+ return false;
+}
+
+static bool
+hns3_get_sve_support(void)
+{
+#if defined(RTE_HAS_SVE_ACLE)
+ if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_256)
+ return false;
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SVE))
+ return true;
+#endif
+ return false;
+}
+
+static eth_rx_burst_t
+hns3_get_rx_function(struct rte_eth_dev *dev)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ uint64_t offloads = dev->data->dev_conf.rxmode.offloads;
+ bool vec_allowed, sve_allowed, simple_allowed;
+ bool vec_support;
+
+ vec_support = hns3_rx_check_vec_support(dev) == 0;
+ vec_allowed = vec_support && hns3_get_default_vec_support();
+ sve_allowed = vec_support && hns3_get_sve_support();
+ simple_allowed = !dev->data->scattered_rx &&
+ (offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) == 0;
+
+ if (hns->rx_func_hint == HNS3_IO_FUNC_HINT_VEC && vec_allowed)
+ return hns3_recv_pkts_vec;
+ if (hns->rx_func_hint == HNS3_IO_FUNC_HINT_SVE && sve_allowed)
+ return hns3_recv_pkts_vec_sve;
+ if (hns->rx_func_hint == HNS3_IO_FUNC_HINT_SIMPLE && simple_allowed)
+ return hns3_recv_pkts_simple;
+ if (hns->rx_func_hint == HNS3_IO_FUNC_HINT_COMMON)
+ return hns3_recv_scattered_pkts;
+
+ if (vec_allowed)
+ return hns3_recv_pkts_vec;
+ if (simple_allowed)
+ return hns3_recv_pkts_simple;
+
+ return hns3_recv_scattered_pkts;
+}
+
+static int
+hns3_tx_queue_conf_check(struct hns3_hw *hw, const struct rte_eth_txconf *conf,
+ uint16_t nb_desc, uint16_t *tx_rs_thresh,
+ uint16_t *tx_free_thresh, uint16_t idx)
+{
+#define HNS3_TX_RS_FREE_THRESH_GAP 8
+ uint16_t rs_thresh, free_thresh, fast_free_thresh;
+
+ if (nb_desc > HNS3_MAX_RING_DESC || nb_desc < HNS3_MIN_RING_DESC ||
+ nb_desc % HNS3_ALIGN_RING_DESC) {
+ hns3_err(hw, "number (%u) of tx descriptors is invalid",
+ nb_desc);
+ return -EINVAL;
+ }
+
+ rs_thresh = (conf->tx_rs_thresh > 0) ?
+ conf->tx_rs_thresh : HNS3_DEFAULT_TX_RS_THRESH;
+ free_thresh = (conf->tx_free_thresh > 0) ?
+ conf->tx_free_thresh : HNS3_DEFAULT_TX_FREE_THRESH;
+ if (rs_thresh + free_thresh > nb_desc || nb_desc % rs_thresh ||
+ rs_thresh >= nb_desc - HNS3_TX_RS_FREE_THRESH_GAP ||
+ free_thresh >= nb_desc - HNS3_TX_RS_FREE_THRESH_GAP) {
+ hns3_err(hw, "tx_rs_thresh (%u) tx_free_thresh (%u) nb_desc "
+ "(%u) of tx descriptors for port=%u queue=%u check "
+ "fail!",
+ rs_thresh, free_thresh, nb_desc, hw->data->port_id,
+ idx);
+ return -EINVAL;
+ }
+
+ if (conf->tx_free_thresh == 0) {
+ /* Fast free Tx memory buffer to improve cache hit rate */
+ fast_free_thresh = nb_desc - rs_thresh;
+ if (fast_free_thresh >=
+ HNS3_TX_FAST_FREE_AHEAD + HNS3_DEFAULT_TX_FREE_THRESH)
+ free_thresh = fast_free_thresh -
+ HNS3_TX_FAST_FREE_AHEAD;
+ }
+
+ *tx_rs_thresh = rs_thresh;
+ *tx_free_thresh = free_thresh;
+ return 0;
+}
+
+static void *
+hns3_tx_push_get_queue_tail_reg(struct rte_eth_dev *dev, uint16_t queue_id)
+{
+#define HNS3_TX_PUSH_TQP_REGION_SIZE 0x10000
+#define HNS3_TX_PUSH_QUICK_DOORBELL_OFFSET 64
+#define HNS3_TX_PUSH_PCI_BAR_INDEX 4
+
+ struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
+ uint8_t bar_id = HNS3_TX_PUSH_PCI_BAR_INDEX;
+
+ /*
+ * If device support Tx push then its PCIe bar45 must exist, and DPDK
+ * framework will mmap the bar45 default in PCI probe stage.
+ *
+ * In the bar45, the first half is for RoCE (RDMA over Converged
+ * Ethernet), and the second half is for NIC, every TQP occupy 64KB.
+ *
+ * The quick doorbell located at 64B offset in the TQP region.
+ */
+ return (char *)pci_dev->mem_resource[bar_id].addr +
+ (pci_dev->mem_resource[bar_id].len >> 1) +
+ HNS3_TX_PUSH_TQP_REGION_SIZE * queue_id +
+ HNS3_TX_PUSH_QUICK_DOORBELL_OFFSET;
+}
+
+void
+hns3_tx_push_init(struct rte_eth_dev *dev)
+{
+ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ volatile uint32_t *reg;
+ uint32_t val;
+
+ if (!hns3_dev_get_support(hw, TX_PUSH))
+ return;
+
+ reg = (volatile uint32_t *)hns3_tx_push_get_queue_tail_reg(dev, 0);
+ /*
+ * Because the size of bar45 is about 8GB size, it may take a long time
+ * to do the page fault in Tx process when work with vfio-pci, so use
+ * one read operation to make kernel setup page table mapping for bar45
+ * in the init stage.
+ * Note: the bar45 is readable but the result is all 1.
+ */
+ val = *reg;
+ RTE_SET_USED(val);
+}
+
+static void
+hns3_tx_push_queue_init(struct rte_eth_dev *dev,
+ uint16_t queue_id,
+ struct hns3_tx_queue *txq)
+{
+ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ if (!hns3_dev_get_support(hw, TX_PUSH)) {
+ txq->tx_push_enable = false;
+ return;
+ }
+
+ txq->io_tail_reg = (volatile void *)hns3_tx_push_get_queue_tail_reg(dev,
+ queue_id);
+ txq->tx_push_enable = true;
+}
+
+int
+hns3_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
+ unsigned int socket_id, const struct rte_eth_txconf *conf)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ uint16_t tx_rs_thresh, tx_free_thresh;
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_queue_info q_info;
+ struct hns3_tx_queue *txq;
+ int tx_entry_len;
+ int ret;
+
+ ret = hns3_tx_queue_conf_check(hw, conf, nb_desc,
+ &tx_rs_thresh, &tx_free_thresh, idx);
+ if (ret)
+ return ret;
+
+ if (dev->data->tx_queues[idx] != NULL) {
+ hns3_tx_queue_release(dev->data->tx_queues[idx]);
+ dev->data->tx_queues[idx] = NULL;
+ }
+
+ q_info.idx = idx;
+ q_info.socket_id = socket_id;
+ q_info.nb_desc = nb_desc;
+ q_info.type = "hns3 TX queue";
+ q_info.ring_name = "tx_ring";
+ txq = hns3_alloc_txq_and_dma_zone(dev, &q_info);
+ if (txq == NULL) {
+ hns3_err(hw,
+ "Failed to alloc mem and reserve DMA mem for tx ring!");
+ return -ENOMEM;
+ }
+
+ txq->tx_deferred_start = conf->tx_deferred_start;
+ if (txq->tx_deferred_start && !hns3_dev_get_support(hw, INDEP_TXRX)) {
+ hns3_warn(hw, "deferred start is not supported.");
+ txq->tx_deferred_start = false;
+ }
+
+ tx_entry_len = sizeof(struct hns3_entry) * txq->nb_tx_desc;
+ txq->sw_ring = rte_zmalloc_socket("hns3 TX sw ring", tx_entry_len,
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (txq->sw_ring == NULL) {
+ hns3_err(hw, "Failed to allocate memory for tx sw ring!");
+ hns3_tx_queue_release(txq);
+ return -ENOMEM;
+ }
+
+ txq->hns = hns;
+ txq->next_to_use = 0;
+ txq->next_to_clean = 0;
+ txq->tx_bd_ready = txq->nb_tx_desc - 1;
+ txq->tx_free_thresh = tx_free_thresh;
+ txq->tx_rs_thresh = tx_rs_thresh;
+ txq->free = rte_zmalloc_socket("hns3 TX mbuf free array",
+ sizeof(struct rte_mbuf *) * txq->tx_rs_thresh,
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (!txq->free) {
+ hns3_err(hw, "failed to allocate tx mbuf free array!");
+ hns3_tx_queue_release(txq);
+ return -ENOMEM;
+ }
+
+ txq->port_id = dev->data->port_id;
+ /*
+ * For hns3 PF device, if the VLAN mode is HW_SHIFT_AND_DISCARD_MODE,
+ * the pvid_sw_shift_en in the queue struct should not be changed,
+ * because PVID-related operations do not need to be processed by PMD.
+ * For hns3 VF device, whether it needs to process PVID depends
+ * on the configuration of PF kernel mode netdev driver. And the
+ * related PF configuration is delivered through the mailbox and finally
+ * reflected in port_base_vlan_cfg.
+ */
+ if (hns->is_vf || hw->vlan_mode == HNS3_SW_SHIFT_AND_DISCARD_MODE)
+ txq->pvid_sw_shift_en = hw->port_base_vlan_cfg.state ==
+ HNS3_PORT_BASE_VLAN_ENABLE;
+ else
+ txq->pvid_sw_shift_en = false;
+ txq->max_non_tso_bd_num = hw->max_non_tso_bd_num;
+ txq->configured = true;
+ txq->io_base = (void *)((char *)hw->io_base +
+ hns3_get_tqp_reg_offset(idx));
+ txq->io_tail_reg = (volatile void *)((char *)txq->io_base +
+ HNS3_RING_TX_TAIL_REG);
+ txq->min_tx_pkt_len = hw->min_tx_pkt_len;
+ txq->tso_mode = hw->tso_mode;
+ txq->udp_cksum_mode = hw->udp_cksum_mode;
+ txq->mbuf_fast_free_en = !!(dev->data->dev_conf.txmode.offloads &
+ RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE);
+ memset(&txq->basic_stats, 0, sizeof(struct hns3_tx_basic_stats));
+ memset(&txq->dfx_stats, 0, sizeof(struct hns3_tx_dfx_stats));
+
+ /*
+ * Call hns3_tx_push_queue_init after assigned io_tail_reg field because
+ * it may overwrite the io_tail_reg field.
+ */
+ hns3_tx_push_queue_init(dev, idx, txq);
+
+ rte_spinlock_lock(&hw->lock);
+ dev->data->tx_queues[idx] = txq;
+ rte_spinlock_unlock(&hw->lock);
+
+ return 0;
+}
+
+static int
+hns3_tx_free_useless_buffer(struct hns3_tx_queue *txq)
+{
+ uint16_t tx_next_clean = txq->next_to_clean;
+ uint16_t tx_next_use = txq->next_to_use;
+ struct hns3_entry *tx_entry = &txq->sw_ring[tx_next_clean];
+ struct hns3_desc *desc = &txq->tx_ring[tx_next_clean];
+ int i;
+
+ if (tx_next_use >= tx_next_clean &&
+ tx_next_use < tx_next_clean + txq->tx_rs_thresh)
+ return -1;
+
+ /*
+ * All mbufs can be released only when the VLD bits of all
+ * descriptors in a batch are cleared.
+ */
+ for (i = 0; i < txq->tx_rs_thresh; i++) {
+ if (desc[i].tx.tp_fe_sc_vld_ra_ri &
+ rte_le_to_cpu_16(BIT(HNS3_TXD_VLD_B)))
+ return -1;
+ }
+
+ for (i = 0; i < txq->tx_rs_thresh; i++) {
+ rte_pktmbuf_free_seg(tx_entry[i].mbuf);
+ tx_entry[i].mbuf = NULL;
+ }
+
+ /* Update numbers of available descriptor due to buffer freed */
+ txq->tx_bd_ready += txq->tx_rs_thresh;
+ txq->next_to_clean += txq->tx_rs_thresh;
+ if (txq->next_to_clean >= txq->nb_tx_desc)
+ txq->next_to_clean = 0;
+
+ return 0;
+}
+
+static inline int
+hns3_tx_free_required_buffer(struct hns3_tx_queue *txq, uint16_t required_bds)
+{
+ while (required_bds > txq->tx_bd_ready) {
+ if (hns3_tx_free_useless_buffer(txq) != 0)
+ return -1;
+ }
+ return 0;