+void
+hns3_dev_release_mbufs(struct hns3_adapter *hns)
+{
+ struct rte_eth_dev_data *dev_data = hns->hw.data;
+ struct hns3_rx_queue *rxq;
+ struct hns3_tx_queue *txq;
+ int i;
+
+ if (dev_data->rx_queues)
+ for (i = 0; i < dev_data->nb_rx_queues; i++) {
+ rxq = dev_data->rx_queues[i];
+ if (rxq == NULL)
+ continue;
+ hns3_rx_queue_release_mbufs(rxq);
+ }
+
+ if (dev_data->tx_queues)
+ for (i = 0; i < dev_data->nb_tx_queues; i++) {
+ txq = dev_data->tx_queues[i];
+ if (txq == NULL)
+ continue;
+ hns3_tx_queue_release_mbufs(txq);
+ }
+}
+
+static int
+hns3_rx_buf_len_calc(struct rte_mempool *mp, uint16_t *rx_buf_len)
+{
+ uint16_t vld_buf_size;
+ uint16_t num_hw_specs;
+ uint16_t i;
+
+ /*
+ * hns3 network engine only support to set 4 typical specification, and
+ * different buffer size will affect the max packet_len and the max
+ * number of segmentation when hw gro is turned on in receive side. The
+ * relationship between them is as follows:
+ * rx_buf_size | max_gro_pkt_len | max_gro_nb_seg
+ * ---------------------|-------------------|----------------
+ * HNS3_4K_BD_BUF_SIZE | 60KB | 15
+ * HNS3_2K_BD_BUF_SIZE | 62KB | 31
+ * HNS3_1K_BD_BUF_SIZE | 63KB | 63
+ * HNS3_512_BD_BUF_SIZE | 31.5KB | 63
+ */
+ static const uint16_t hw_rx_buf_size[] = {
+ HNS3_4K_BD_BUF_SIZE,
+ HNS3_2K_BD_BUF_SIZE,
+ HNS3_1K_BD_BUF_SIZE,
+ HNS3_512_BD_BUF_SIZE
+ };
+
+ vld_buf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
+ RTE_PKTMBUF_HEADROOM);
+ if (vld_buf_size < HNS3_MIN_BD_BUF_SIZE)
+ return -EINVAL;
+
+ num_hw_specs = RTE_DIM(hw_rx_buf_size);
+ for (i = 0; i < num_hw_specs; i++) {
+ if (vld_buf_size >= hw_rx_buf_size[i]) {
+ *rx_buf_len = hw_rx_buf_size[i];
+ break;
+ }
+ }
+ return 0;
+}
+
+static int
+hns3_rxq_conf_runtime_check(struct hns3_hw *hw, uint16_t buf_size,
+ uint16_t nb_desc)
+{
+ struct rte_eth_dev *dev = &rte_eth_devices[hw->data->port_id];
+ eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
+ uint32_t frame_size = dev->data->mtu + HNS3_ETH_OVERHEAD;
+ uint16_t min_vec_bds;
+
+ /*
+ * HNS3 hardware network engine set scattered as default. If the driver
+ * is not work in scattered mode and the pkts greater than buf_size
+ * but smaller than frame size will be distributed to multiple BDs.
+ * Driver cannot handle this situation.
+ */
+ if (!hw->data->scattered_rx && frame_size > buf_size) {
+ hns3_err(hw, "frame size is not allowed to be set greater "
+ "than rx_buf_len if scattered is off.");
+ return -EINVAL;
+ }
+
+ if (pkt_burst == hns3_recv_pkts_vec) {
+ min_vec_bds = HNS3_DEFAULT_RXQ_REARM_THRESH +
+ HNS3_DEFAULT_RX_BURST;
+ if (nb_desc < min_vec_bds ||
+ nb_desc % HNS3_DEFAULT_RXQ_REARM_THRESH) {
+ hns3_err(hw, "if Rx burst mode is vector, "
+ "number of descriptor is required to be "
+ "bigger than min vector bds:%u, and could be "
+ "divided by rxq rearm thresh:%u.",
+ min_vec_bds, HNS3_DEFAULT_RXQ_REARM_THRESH);
+ return -EINVAL;
+ }
+ }
+ return 0;
+}
+
+static int
+hns3_rx_queue_conf_check(struct hns3_hw *hw, const struct rte_eth_rxconf *conf,
+ struct rte_mempool *mp, uint16_t nb_desc,
+ uint16_t *buf_size)
+{
+ int ret;
+
+ if (nb_desc > HNS3_MAX_RING_DESC || nb_desc < HNS3_MIN_RING_DESC ||
+ nb_desc % HNS3_ALIGN_RING_DESC) {
+ hns3_err(hw, "Number (%u) of rx descriptors is invalid",
+ nb_desc);
+ return -EINVAL;
+ }
+
+ if (conf->rx_drop_en == 0)
+ hns3_warn(hw, "if no descriptors available, packets are always "
+ "dropped and rx_drop_en (1) is fixed on");
+
+ if (hns3_rx_buf_len_calc(mp, buf_size)) {
+ hns3_err(hw, "rxq mbufs' data room size (%u) is not enough! "
+ "minimal data room size (%u).",
+ rte_pktmbuf_data_room_size(mp),
+ HNS3_MIN_BD_BUF_SIZE + RTE_PKTMBUF_HEADROOM);
+ return -EINVAL;
+ }
+
+ if (hw->data->dev_started) {
+ ret = hns3_rxq_conf_runtime_check(hw, *buf_size, nb_desc);
+ if (ret) {
+ hns3_err(hw, "Rx queue runtime setup fail.");
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+uint32_t
+hns3_get_tqp_reg_offset(uint16_t queue_id)
+{
+ uint32_t reg_offset;
+
+ /* Need an extend offset to config queue > 1024 */
+ if (queue_id < HNS3_MIN_EXTEND_QUEUE_ID)
+ reg_offset = HNS3_TQP_REG_OFFSET + queue_id * HNS3_TQP_REG_SIZE;
+ else
+ reg_offset = HNS3_TQP_REG_OFFSET + HNS3_TQP_EXT_REG_OFFSET +
+ (queue_id - HNS3_MIN_EXTEND_QUEUE_ID) *
+ HNS3_TQP_REG_SIZE;
+
+ return reg_offset;
+}
+
+int
+hns3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t nb_desc,
+ unsigned int socket_id, const struct rte_eth_rxconf *conf,
+ struct rte_mempool *mp)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_queue_info q_info;
+ struct hns3_rx_queue *rxq;
+ uint16_t rx_buf_size;
+ int rx_entry_len;
+ int ret;
+
+ ret = hns3_rx_queue_conf_check(hw, conf, mp, nb_desc, &rx_buf_size);
+ if (ret)
+ return ret;
+
+ if (dev->data->rx_queues[idx]) {
+ hns3_rx_queue_release(dev->data->rx_queues[idx]);
+ dev->data->rx_queues[idx] = NULL;
+ }
+
+ q_info.idx = idx;
+ q_info.socket_id = socket_id;
+ q_info.nb_desc = nb_desc;
+ q_info.type = "hns3 RX queue";
+ q_info.ring_name = "rx_ring";
+
+ rxq = hns3_alloc_rxq_and_dma_zone(dev, &q_info);
+ if (rxq == NULL) {
+ hns3_err(hw,
+ "Failed to alloc mem and reserve DMA mem for rx ring!");
+ return -ENOMEM;
+ }
+
+ rxq->hns = hns;
+ rxq->ptype_tbl = &hns->ptype_tbl;
+ rxq->mb_pool = mp;
+ rxq->rx_free_thresh = (conf->rx_free_thresh > 0) ?
+ conf->rx_free_thresh : HNS3_DEFAULT_RX_FREE_THRESH;
+
+ rxq->rx_deferred_start = conf->rx_deferred_start;
+ if (rxq->rx_deferred_start && !hns3_dev_get_support(hw, INDEP_TXRX)) {
+ hns3_warn(hw, "deferred start is not supported.");
+ rxq->rx_deferred_start = false;
+ }
+
+ rx_entry_len = (rxq->nb_rx_desc + HNS3_DEFAULT_RX_BURST) *
+ sizeof(struct hns3_entry);
+ rxq->sw_ring = rte_zmalloc_socket("hns3 RX sw ring", rx_entry_len,
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (rxq->sw_ring == NULL) {
+ hns3_err(hw, "Failed to allocate memory for rx sw ring!");
+ hns3_rx_queue_release(rxq);
+ return -ENOMEM;
+ }
+
+ rxq->next_to_use = 0;
+ rxq->rx_free_hold = 0;
+ rxq->rx_rearm_start = 0;
+ rxq->rx_rearm_nb = 0;
+ rxq->pkt_first_seg = NULL;
+ rxq->pkt_last_seg = NULL;
+ rxq->port_id = dev->data->port_id;
+ /*
+ * For hns3 PF device, if the VLAN mode is HW_SHIFT_AND_DISCARD_MODE,
+ * the pvid_sw_discard_en in the queue struct should not be changed,
+ * because PVID-related operations do not need to be processed by PMD.
+ * For hns3 VF device, whether it needs to process PVID depends
+ * on the configuration of PF kernel mode netdevice driver. And the
+ * related PF configuration is delivered through the mailbox and finally
+ * reflected in port_base_vlan_cfg.
+ */
+ if (hns->is_vf || hw->vlan_mode == HNS3_SW_SHIFT_AND_DISCARD_MODE)
+ rxq->pvid_sw_discard_en = hw->port_base_vlan_cfg.state ==
+ HNS3_PORT_BASE_VLAN_ENABLE;
+ else
+ rxq->pvid_sw_discard_en = false;
+ rxq->ptype_en = hns3_dev_get_support(hw, RXD_ADV_LAYOUT) ? true : false;
+ rxq->configured = true;
+ rxq->io_base = (void *)((char *)hw->io_base + HNS3_TQP_REG_OFFSET +
+ idx * HNS3_TQP_REG_SIZE);
+ rxq->io_base = (void *)((char *)hw->io_base +
+ hns3_get_tqp_reg_offset(idx));
+ rxq->io_head_reg = (volatile void *)((char *)rxq->io_base +
+ HNS3_RING_RX_HEAD_REG);
+ rxq->rx_buf_len = rx_buf_size;
+ memset(&rxq->basic_stats, 0, sizeof(struct hns3_rx_basic_stats));
+ memset(&rxq->err_stats, 0, sizeof(struct hns3_rx_bd_errors_stats));
+ memset(&rxq->dfx_stats, 0, sizeof(struct hns3_rx_dfx_stats));
+
+ /* CRC len set here is used for amending packet length */
+ if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC)
+ rxq->crc_len = RTE_ETHER_CRC_LEN;
+ else
+ rxq->crc_len = 0;
+
+ rxq->bulk_mbuf_num = 0;
+
+ rte_spinlock_lock(&hw->lock);
+ dev->data->rx_queues[idx] = rxq;
+ rte_spinlock_unlock(&hw->lock);
+
+ return 0;
+}
+
+void
+hns3_rx_scattered_reset(struct rte_eth_dev *dev)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_hw *hw = &hns->hw;
+
+ hw->rx_buf_len = 0;
+ dev->data->scattered_rx = false;
+}
+
+void
+hns3_rx_scattered_calc(struct rte_eth_dev *dev)
+{
+ struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_hw *hw = &hns->hw;
+ struct hns3_rx_queue *rxq;
+ uint32_t queue_id;
+
+ if (dev->data->rx_queues == NULL)
+ return;
+
+ for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
+ rxq = dev->data->rx_queues[queue_id];
+ if (hw->rx_buf_len == 0)
+ hw->rx_buf_len = rxq->rx_buf_len;
+ else
+ hw->rx_buf_len = RTE_MIN(hw->rx_buf_len,
+ rxq->rx_buf_len);
+ }
+
+ if (dev_conf->rxmode.offloads & RTE_ETH_RX_OFFLOAD_SCATTER ||
+ dev->data->mtu + HNS3_ETH_OVERHEAD > hw->rx_buf_len)
+ dev->data->scattered_rx = true;
+}
+
+const uint32_t *
+hns3_dev_supported_ptypes_get(struct rte_eth_dev *dev)
+{
+ static const uint32_t ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L2_ETHER_LLDP,
+ RTE_PTYPE_L2_ETHER_ARP,
+ RTE_PTYPE_L3_IPV4,
+ RTE_PTYPE_L3_IPV4_EXT,
+ RTE_PTYPE_L3_IPV6,
+ RTE_PTYPE_L3_IPV6_EXT,
+ RTE_PTYPE_L4_IGMP,
+ RTE_PTYPE_L4_ICMP,
+ RTE_PTYPE_L4_SCTP,
+ RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRE,
+ RTE_PTYPE_INNER_L2_ETHER,
+ RTE_PTYPE_INNER_L3_IPV4,
+ RTE_PTYPE_INNER_L3_IPV6,
+ RTE_PTYPE_INNER_L3_IPV4_EXT,
+ RTE_PTYPE_INNER_L3_IPV6_EXT,
+ RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_INNER_L4_SCTP,
+ RTE_PTYPE_INNER_L4_ICMP,
+ RTE_PTYPE_TUNNEL_VXLAN,
+ RTE_PTYPE_TUNNEL_NVGRE,
+ RTE_PTYPE_UNKNOWN
+ };
+ static const uint32_t adv_layout_ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L2_ETHER_TIMESYNC,
+ RTE_PTYPE_L2_ETHER_LLDP,
+ RTE_PTYPE_L2_ETHER_ARP,
+ RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_L4_FRAG,
+ RTE_PTYPE_L4_NONFRAG,
+ RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L4_SCTP,
+ RTE_PTYPE_L4_IGMP,
+ RTE_PTYPE_L4_ICMP,
+ RTE_PTYPE_TUNNEL_GRE,
+ RTE_PTYPE_TUNNEL_GRENAT,
+ RTE_PTYPE_INNER_L2_ETHER,
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_INNER_L4_FRAG,
+ RTE_PTYPE_INNER_L4_ICMP,
+ RTE_PTYPE_INNER_L4_NONFRAG,
+ RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_INNER_L4_SCTP,
+ RTE_PTYPE_INNER_L4_ICMP,
+ RTE_PTYPE_UNKNOWN
+ };
+ struct hns3_hw *hw = HNS3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+ if (dev->rx_pkt_burst == hns3_recv_pkts_simple ||
+ dev->rx_pkt_burst == hns3_recv_scattered_pkts ||
+ dev->rx_pkt_burst == hns3_recv_pkts_vec ||
+ dev->rx_pkt_burst == hns3_recv_pkts_vec_sve) {
+ if (hns3_dev_get_support(hw, RXD_ADV_LAYOUT))
+ return adv_layout_ptypes;
+ else
+ return ptypes;
+ }
+
+ return NULL;
+}
+
+static void
+hns3_init_non_tunnel_ptype_tbl(struct hns3_ptype_table *tbl)
+{
+ tbl->l3table[0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
+ tbl->l3table[1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
+ tbl->l3table[2] = RTE_PTYPE_L2_ETHER_ARP;
+ tbl->l3table[4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT;
+ tbl->l3table[5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT;
+ tbl->l3table[6] = RTE_PTYPE_L2_ETHER_LLDP;
+
+ tbl->l4table[0] = RTE_PTYPE_L4_UDP;
+ tbl->l4table[1] = RTE_PTYPE_L4_TCP;
+ tbl->l4table[2] = RTE_PTYPE_TUNNEL_GRE;
+ tbl->l4table[3] = RTE_PTYPE_L4_SCTP;
+ tbl->l4table[4] = RTE_PTYPE_L4_IGMP;
+ tbl->l4table[5] = RTE_PTYPE_L4_ICMP;
+}
+
+static void
+hns3_init_tunnel_ptype_tbl(struct hns3_ptype_table *tbl)
+{
+ tbl->inner_l3table[0] = RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4;
+ tbl->inner_l3table[1] = RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6;
+ /* There is not a ptype for inner ARP/RARP */
+ tbl->inner_l3table[2] = RTE_PTYPE_UNKNOWN;
+ tbl->inner_l3table[3] = RTE_PTYPE_UNKNOWN;
+ tbl->inner_l3table[4] = RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT;
+ tbl->inner_l3table[5] = RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT;
+
+ tbl->inner_l4table[0] = RTE_PTYPE_INNER_L4_UDP;
+ tbl->inner_l4table[1] = RTE_PTYPE_INNER_L4_TCP;
+ /* There is not a ptype for inner GRE */
+ tbl->inner_l4table[2] = RTE_PTYPE_UNKNOWN;
+ tbl->inner_l4table[3] = RTE_PTYPE_INNER_L4_SCTP;
+ /* There is not a ptype for inner IGMP */
+ tbl->inner_l4table[4] = RTE_PTYPE_UNKNOWN;
+ tbl->inner_l4table[5] = RTE_PTYPE_INNER_L4_ICMP;
+
+ tbl->ol3table[0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
+ tbl->ol3table[1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
+ tbl->ol3table[2] = RTE_PTYPE_UNKNOWN;
+ tbl->ol3table[3] = RTE_PTYPE_UNKNOWN;
+ tbl->ol3table[4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT;
+ tbl->ol3table[5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT;
+
+ tbl->ol4table[0] = RTE_PTYPE_UNKNOWN;
+ tbl->ol4table[1] = RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN;
+ tbl->ol4table[2] = RTE_PTYPE_TUNNEL_NVGRE;
+}
+
+static void
+hns3_init_adv_layout_ptype(struct hns3_ptype_table *tbl)
+{
+ uint32_t *ptype = tbl->ptype;
+
+ /* Non-tunnel L2 */
+ ptype[1] = RTE_PTYPE_L2_ETHER_ARP;
+ ptype[3] = RTE_PTYPE_L2_ETHER_LLDP;
+ ptype[8] = RTE_PTYPE_L2_ETHER_TIMESYNC;
+
+ /* Non-tunnel IPv4 */
+ ptype[17] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_FRAG;
+ ptype[18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[19] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRE;
+ ptype[22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_SCTP;
+ ptype[23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_IGMP;
+ ptype[24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_ICMP;
+ /* The next ptype is PTP over IPv4 + UDP */
+ ptype[25] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+
+ /* IPv4 --> GRE/Teredo/VXLAN */
+ ptype[29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv4 --> GRE/Teredo/VXLAN --> MAC */
+ ptype[30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
+ ptype[31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ /* The next ptype's inner L4 is IGMP */
+ ptype[36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN;
+ ptype[37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
+ ptype[39] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ /* The next ptype's inner L4 is IGMP */
+ ptype[44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN;
+ ptype[45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* Non-tunnel IPv6 */
+ ptype[111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_FRAG;
+ ptype[112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_NONFRAG;
+ ptype[113] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+ ptype[114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_TCP;
+ ptype[115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRE;
+ ptype[116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_SCTP;
+ ptype[117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_IGMP;
+ ptype[118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_ICMP;
+ /* Special for PTP over IPv6 + UDP */
+ ptype[119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_L4_UDP;
+
+ /* IPv6 --> GRE/Teredo/VXLAN */
+ ptype[123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT;
+ /* IPv6 --> GRE/Teredo/VXLAN --> MAC */
+ ptype[124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER;
+
+ /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
+ ptype[125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[128] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ /* The next ptype's inner L4 is IGMP */
+ ptype[130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN;
+ ptype[131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+
+ /* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
+ ptype[133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_FRAG;
+ ptype[134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_NONFRAG;
+ ptype[135] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_UDP;
+ ptype[136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_TCP;
+ ptype[137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_SCTP;
+ /* The next ptype's inner L4 is IGMP */
+ ptype[138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN;
+ ptype[139] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
+ RTE_PTYPE_INNER_L4_ICMP;
+}
+
+void
+hns3_init_rx_ptype_tble(struct rte_eth_dev *dev)
+{
+ struct hns3_adapter *hns = dev->data->dev_private;
+ struct hns3_ptype_table *tbl = &hns->ptype_tbl;
+
+ memset(tbl, 0, sizeof(*tbl));
+
+ hns3_init_non_tunnel_ptype_tbl(tbl);
+ hns3_init_tunnel_ptype_tbl(tbl);
+ hns3_init_adv_layout_ptype(tbl);
+}
+
+static inline void
+hns3_rxd_to_vlan_tci(struct hns3_rx_queue *rxq, struct rte_mbuf *mb,
+ uint32_t l234_info, const struct hns3_desc *rxd)
+{
+#define HNS3_STRP_STATUS_NUM 0x4
+
+#define HNS3_NO_STRP_VLAN_VLD 0x0
+#define HNS3_INNER_STRP_VLAN_VLD 0x1
+#define HNS3_OUTER_STRP_VLAN_VLD 0x2
+ uint32_t strip_status;
+ uint32_t report_mode;
+
+ /*
+ * Since HW limitation, the vlan tag will always be inserted into RX
+ * descriptor when strip the tag from packet, driver needs to determine
+ * reporting which tag to mbuf according to the PVID configuration
+ * and vlan striped status.
+ */
+ static const uint32_t report_type[][HNS3_STRP_STATUS_NUM] = {
+ {
+ HNS3_NO_STRP_VLAN_VLD,
+ HNS3_OUTER_STRP_VLAN_VLD,
+ HNS3_INNER_STRP_VLAN_VLD,
+ HNS3_OUTER_STRP_VLAN_VLD
+ },
+ {
+ HNS3_NO_STRP_VLAN_VLD,
+ HNS3_NO_STRP_VLAN_VLD,
+ HNS3_NO_STRP_VLAN_VLD,
+ HNS3_INNER_STRP_VLAN_VLD
+ }
+ };
+ strip_status = hns3_get_field(l234_info, HNS3_RXD_STRP_TAGP_M,
+ HNS3_RXD_STRP_TAGP_S);
+ report_mode = report_type[rxq->pvid_sw_discard_en][strip_status];
+ switch (report_mode) {
+ case HNS3_NO_STRP_VLAN_VLD:
+ mb->vlan_tci = 0;
+ return;
+ case HNS3_INNER_STRP_VLAN_VLD:
+ mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
+ mb->vlan_tci = rte_le_to_cpu_16(rxd->rx.vlan_tag);
+ return;
+ case HNS3_OUTER_STRP_VLAN_VLD:
+ mb->ol_flags |= RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED;
+ mb->vlan_tci = rte_le_to_cpu_16(rxd->rx.ot_vlan_tag);
+ return;
+ default:
+ mb->vlan_tci = 0;
+ return;
+ }
+}
+
+static inline void
+recalculate_data_len(struct rte_mbuf *first_seg, struct rte_mbuf *last_seg,
+ struct rte_mbuf *rxm, struct hns3_rx_queue *rxq,
+ uint16_t data_len)
+{
+ uint8_t crc_len = rxq->crc_len;
+
+ if (data_len <= crc_len) {
+ rte_pktmbuf_free_seg(rxm);
+ first_seg->nb_segs--;
+ last_seg->data_len = (uint16_t)(last_seg->data_len -
+ (crc_len - data_len));
+ last_seg->next = NULL;
+ } else
+ rxm->data_len = (uint16_t)(data_len - crc_len);
+}
+
+static inline struct rte_mbuf *
+hns3_rx_alloc_buffer(struct hns3_rx_queue *rxq)
+{
+ int ret;
+
+ if (likely(rxq->bulk_mbuf_num > 0))
+ return rxq->bulk_mbuf[--rxq->bulk_mbuf_num];
+
+ ret = rte_mempool_get_bulk(rxq->mb_pool, (void **)rxq->bulk_mbuf,
+ HNS3_BULK_ALLOC_MBUF_NUM);
+ if (likely(ret == 0)) {
+ rxq->bulk_mbuf_num = HNS3_BULK_ALLOC_MBUF_NUM;
+ return rxq->bulk_mbuf[--rxq->bulk_mbuf_num];
+ } else
+ return rte_mbuf_raw_alloc(rxq->mb_pool);
+}
+
+static void
+hns3_rx_ptp_timestamp_handle(struct hns3_rx_queue *rxq, struct rte_mbuf *mbuf,
+ uint64_t timestamp)
+{
+ struct hns3_pf *pf = HNS3_DEV_PRIVATE_TO_PF(rxq->hns);
+
+ mbuf->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP |
+ RTE_MBUF_F_RX_IEEE1588_TMST;
+ if (hns3_timestamp_rx_dynflag > 0) {
+ *RTE_MBUF_DYNFIELD(mbuf, hns3_timestamp_dynfield_offset,
+ rte_mbuf_timestamp_t *) = timestamp;
+ mbuf->ol_flags |= hns3_timestamp_rx_dynflag;
+ }
+
+ pf->rx_timestamp = timestamp;
+}
+
+uint16_t
+hns3_recv_pkts_simple(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ volatile struct hns3_desc *rx_ring; /* RX ring (desc) */
+ volatile struct hns3_desc *rxdp; /* pointer of the current desc */
+ struct hns3_rx_queue *rxq; /* RX queue */
+ struct hns3_entry *sw_ring;
+ struct hns3_entry *rxe;
+ struct hns3_desc rxd;
+ struct rte_mbuf *nmb; /* pointer of the new mbuf */
+ struct rte_mbuf *rxm;
+ uint32_t bd_base_info;
+ uint32_t l234_info;
+ uint32_t ol_info;
+ uint64_t dma_addr;
+ uint16_t nb_rx_bd;
+ uint16_t nb_rx;
+ uint16_t rx_id;
+ int ret;
+
+ nb_rx = 0;
+ nb_rx_bd = 0;
+ rxq = rx_queue;
+ rx_ring = rxq->rx_ring;
+ sw_ring = rxq->sw_ring;
+ rx_id = rxq->next_to_use;
+
+ while (nb_rx < nb_pkts) {
+ rxdp = &rx_ring[rx_id];
+ bd_base_info = rte_le_to_cpu_32(rxdp->rx.bd_base_info);
+ if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B))))
+ break;
+
+ rxd = rxdp[(bd_base_info & (1u << HNS3_RXD_VLD_B)) -
+ (1u << HNS3_RXD_VLD_B)];
+
+ nmb = hns3_rx_alloc_buffer(rxq);
+ if (unlikely(nmb == NULL)) {
+ uint16_t port_id;
+
+ port_id = rxq->port_id;
+ rte_eth_devices[port_id].data->rx_mbuf_alloc_failed++;
+ break;
+ }
+
+ nb_rx_bd++;
+ rxe = &sw_ring[rx_id];
+ rx_id++;
+ if (unlikely(rx_id == rxq->nb_rx_desc))
+ rx_id = 0;
+
+ rte_prefetch0(sw_ring[rx_id].mbuf);
+ if ((rx_id & HNS3_RX_RING_PREFETCTH_MASK) == 0) {
+ rte_prefetch0(&rx_ring[rx_id]);
+ rte_prefetch0(&sw_ring[rx_id]);
+ }
+
+ rxm = rxe->mbuf;
+ rxm->ol_flags = 0;
+ rxe->mbuf = nmb;
+
+ if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B)))
+ hns3_rx_ptp_timestamp_handle(rxq, rxm,
+ rte_le_to_cpu_64(rxdp->timestamp));
+
+ dma_addr = rte_mbuf_data_iova_default(nmb);
+ rxdp->addr = rte_cpu_to_le_64(dma_addr);
+ rxdp->rx.bd_base_info = 0;
+
+ rxm->data_off = RTE_PKTMBUF_HEADROOM;
+ rxm->pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.rx.pkt_len)) -
+ rxq->crc_len;
+ rxm->data_len = rxm->pkt_len;
+ rxm->port = rxq->port_id;
+ rxm->hash.rss = rte_le_to_cpu_32(rxd.rx.rss_hash);
+ rxm->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
+ if (unlikely(bd_base_info & BIT(HNS3_RXD_LUM_B))) {
+ rxm->hash.fdir.hi =
+ rte_le_to_cpu_16(rxd.rx.fd_id);
+ rxm->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
+ }
+ rxm->nb_segs = 1;
+ rxm->next = NULL;
+
+ /* Load remained descriptor data and extract necessary fields */
+ l234_info = rte_le_to_cpu_32(rxd.rx.l234_info);
+ ol_info = rte_le_to_cpu_32(rxd.rx.ol_info);
+ ret = hns3_handle_bdinfo(rxq, rxm, bd_base_info, l234_info);
+ if (unlikely(ret))
+ goto pkt_err;
+
+ rxm->packet_type = hns3_rx_calc_ptype(rxq, l234_info, ol_info);
+
+ if (rxm->packet_type == RTE_PTYPE_L2_ETHER_TIMESYNC)
+ rxm->ol_flags |= RTE_MBUF_F_RX_IEEE1588_PTP;
+
+ hns3_rxd_to_vlan_tci(rxq, rxm, l234_info, &rxd);
+
+ /* Increment bytes counter */
+ rxq->basic_stats.bytes += rxm->pkt_len;
+
+ rx_pkts[nb_rx++] = rxm;
+ continue;
+pkt_err:
+ rte_pktmbuf_free(rxm);
+ }
+
+ rxq->next_to_use = rx_id;
+ rxq->rx_free_hold += nb_rx_bd;
+ if (rxq->rx_free_hold > rxq->rx_free_thresh) {
+ hns3_write_reg_opt(rxq->io_head_reg, rxq->rx_free_hold);
+ rxq->rx_free_hold = 0;
+ }
+
+ return nb_rx;
+}
+
+uint16_t
+hns3_recv_scattered_pkts(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ volatile struct hns3_desc *rx_ring; /* RX ring (desc) */
+ volatile struct hns3_desc *rxdp; /* pointer of the current desc */
+ struct hns3_rx_queue *rxq; /* RX queue */
+ struct hns3_entry *sw_ring;
+ struct hns3_entry *rxe;
+ struct rte_mbuf *first_seg;
+ struct rte_mbuf *last_seg;
+ struct hns3_desc rxd;
+ struct rte_mbuf *nmb; /* pointer of the new mbuf */
+ struct rte_mbuf *rxm;
+ struct rte_eth_dev *dev;
+ uint32_t bd_base_info;
+ uint64_t timestamp;
+ uint32_t l234_info;
+ uint32_t gro_size;
+ uint32_t ol_info;
+ uint64_t dma_addr;
+ uint16_t nb_rx_bd;
+ uint16_t nb_rx;
+ uint16_t rx_id;
+ int ret;
+
+ nb_rx = 0;
+ nb_rx_bd = 0;
+ rxq = rx_queue;
+
+ rx_id = rxq->next_to_use;
+ rx_ring = rxq->rx_ring;
+ sw_ring = rxq->sw_ring;
+ first_seg = rxq->pkt_first_seg;
+ last_seg = rxq->pkt_last_seg;
+
+ while (nb_rx < nb_pkts) {
+ rxdp = &rx_ring[rx_id];
+ bd_base_info = rte_le_to_cpu_32(rxdp->rx.bd_base_info);
+ if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B))))
+ break;
+
+ /*
+ * The interactive process between software and hardware of
+ * receiving a new packet in hns3 network engine:
+ * 1. Hardware network engine firstly writes the packet content
+ * to the memory pointed by the 'addr' field of the Rx Buffer
+ * Descriptor, secondly fills the result of parsing the
+ * packet include the valid field into the Rx Buffer
+ * Descriptor in one write operation.
+ * 2. Driver reads the Rx BD's valid field in the loop to check
+ * whether it's valid, if valid then assign a new address to
+ * the addr field, clear the valid field, get the other
+ * information of the packet by parsing Rx BD's other fields,
+ * finally write back the number of Rx BDs processed by the
+ * driver to the HNS3_RING_RX_HEAD_REG register to inform
+ * hardware.
+ * In the above process, the ordering is very important. We must
+ * make sure that CPU read Rx BD's other fields only after the
+ * Rx BD is valid.
+ *
+ * There are two type of re-ordering: compiler re-ordering and
+ * CPU re-ordering under the ARMv8 architecture.
+ * 1. we use volatile to deal with compiler re-ordering, so you
+ * can see that rx_ring/rxdp defined with volatile.
+ * 2. we commonly use memory barrier to deal with CPU
+ * re-ordering, but the cost is high.
+ *
+ * In order to solve the high cost of using memory barrier, we
+ * use the data dependency order under the ARMv8 architecture,
+ * for example:
+ * instr01: load A
+ * instr02: load B <- A
+ * the instr02 will always execute after instr01.
+ *
+ * To construct the data dependency ordering, we use the
+ * following assignment:
+ * rxd = rxdp[(bd_base_info & (1u << HNS3_RXD_VLD_B)) -
+ * (1u<<HNS3_RXD_VLD_B)]
+ * Using gcc compiler under the ARMv8 architecture, the related
+ * assembly code example as follows:
+ * note: (1u << HNS3_RXD_VLD_B) equal 0x10
+ * instr01: ldr w26, [x22, #28] --read bd_base_info
+ * instr02: and w0, w26, #0x10 --calc bd_base_info & 0x10
+ * instr03: sub w0, w0, #0x10 --calc (bd_base_info &
+ * 0x10) - 0x10
+ * instr04: add x0, x22, x0, lsl #5 --calc copy source addr
+ * instr05: ldp x2, x3, [x0]
+ * instr06: stp x2, x3, [x29, #256] --copy BD's [0 ~ 15]B
+ * instr07: ldp x4, x5, [x0, #16]
+ * instr08: stp x4, x5, [x29, #272] --copy BD's [16 ~ 31]B
+ * the instr05~08 depend on x0's value, x0 depent on w26's
+ * value, the w26 is the bd_base_info, this form the data
+ * dependency ordering.
+ * note: if BD is valid, (bd_base_info & (1u<<HNS3_RXD_VLD_B)) -
+ * (1u<<HNS3_RXD_VLD_B) will always zero, so the
+ * assignment is correct.
+ *
+ * So we use the data dependency ordering instead of memory
+ * barrier to improve receive performance.
+ */
+ rxd = rxdp[(bd_base_info & (1u << HNS3_RXD_VLD_B)) -
+ (1u << HNS3_RXD_VLD_B)];
+
+ nmb = hns3_rx_alloc_buffer(rxq);
+ if (unlikely(nmb == NULL)) {
+ dev = &rte_eth_devices[rxq->port_id];
+ dev->data->rx_mbuf_alloc_failed++;
+ break;
+ }
+
+ nb_rx_bd++;
+ rxe = &sw_ring[rx_id];
+ rx_id++;
+ if (unlikely(rx_id == rxq->nb_rx_desc))
+ rx_id = 0;
+
+ rte_prefetch0(sw_ring[rx_id].mbuf);
+ if ((rx_id & HNS3_RX_RING_PREFETCTH_MASK) == 0) {
+ rte_prefetch0(&rx_ring[rx_id]);
+ rte_prefetch0(&sw_ring[rx_id]);
+ }
+
+ rxm = rxe->mbuf;
+ rxe->mbuf = nmb;
+
+ if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B)))
+ timestamp = rte_le_to_cpu_64(rxdp->timestamp);
+
+ dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
+ rxdp->rx.bd_base_info = 0;
+ rxdp->addr = dma_addr;
+
+ if (first_seg == NULL) {